summaryrefslogtreecommitdiff
path: root/gc
diff options
context:
space:
mode:
Diffstat (limited to 'gc')
-rw-r--r--gc/README.md9
-rw-r--r--gc/default/default.c1996
-rw-r--r--gc/gc.h151
-rw-r--r--gc/gc_impl.h22
-rw-r--r--gc/mmtk/Cargo.lock469
-rw-r--r--gc/mmtk/Cargo.toml9
-rw-r--r--gc/mmtk/cbindgen.toml5
-rw-r--r--gc/mmtk/extconf.rb4
-rw-r--r--gc/mmtk/mmtk.c713
-rw-r--r--gc/mmtk/mmtk.h41
-rw-r--r--gc/mmtk/src/abi.rs102
-rw-r--r--gc/mmtk/src/api.rs322
-rw-r--r--gc/mmtk/src/binding.rs10
-rw-r--r--gc/mmtk/src/collection.rs61
-rw-r--r--gc/mmtk/src/heap/cpu_heap_trigger.rs370
-rw-r--r--gc/mmtk/src/heap/mod.rs9
-rw-r--r--gc/mmtk/src/heap/ruby_heap_trigger.rs105
-rw-r--r--gc/mmtk/src/lib.rs36
-rw-r--r--gc/mmtk/src/object_model.rs53
-rw-r--r--gc/mmtk/src/pinning_registry.rs187
-rw-r--r--gc/mmtk/src/scanning.rs72
-rw-r--r--gc/mmtk/src/utils.rs60
-rw-r--r--gc/mmtk/src/weak_proc.rs257
-rw-r--r--gc/wbcheck/extconf.rb3
-rw-r--r--gc/wbcheck/wbcheck.c1936
25 files changed, 5419 insertions, 1583 deletions
diff --git a/gc/README.md b/gc/README.md
index 838d2958db..cb71357973 100644
--- a/gc/README.md
+++ b/gc/README.md
@@ -15,12 +15,17 @@ Two GC implementations are included in Ruby:
> [!IMPORTANT]
> Ruby's modular GC feature is experimental and subject to change. There may be bugs or performance impacts. Use at your own risk.
+### Building Ruby with Modular GC
+
1. Configure Ruby with the `--with-modular-gc=<dir>` option, where `dir` is the directory you want to place the built GC libraries into.
2. Build Ruby as usual.
-3. Build your desired GC implementation with `make modular-gc MODULAR_GC=<impl>`. This will build the GC implementation and place the built library into the `dir` specified in step 1. `impl` can be one of:
+
+### Building GC implementations shipped with Ruby
+
+1. Build your desired GC implementation with `make install-modular-gc MODULAR_GC=<impl>`. This will build the GC implementation and place the built library into the `dir` specified in step 1. `impl` can be one of:
- `default`: The default GC that Ruby ships with.
- `mmtk`: The GC that uses [MMTk](https://www.mmtk.io/) as the back-end. See Ruby-specific details in the [ruby/mmtk](https://github.com/ruby/mmtk) repository.
-4. Run your desired GC implementation by setting the `RUBY_GC_LIBRARY=<lib>` environment variable, where `lib` could be `default`, `mmtk`, or your own implementation (as long as you place it in the `dir` specified in step 1).
+2. Run your desired GC implementation by setting the `RUBY_GC_LIBRARY=<lib>` environment variable, where `lib` could be `default`, `mmtk`, or your own implementation (as long as you place it in the `dir` specified in step 1).
## Modular GC API
diff --git a/gc/default/default.c b/gc/default/default.c
index 919df03440..0027f7a13c 100644
--- a/gc/default/default.c
+++ b/gc/default/default.c
@@ -15,11 +15,15 @@
# include <sys/user.h>
#endif
-#include "internal/bits.h"
-#include "internal/hash.h"
+#ifdef BUILDING_MODULAR_GC
+# define nlz_int64(x) (x == 0 ? 64 : (unsigned int)__builtin_clzll((unsigned long long)x))
+#else
+# include "internal/bits.h"
+#endif
#include "ruby/ruby.h"
#include "ruby/atomic.h"
+#include "ruby_atomic.h"
#include "ruby/debug.h"
#include "ruby/thread.h"
#include "ruby/util.h"
@@ -34,8 +38,26 @@
# include "probes.h"
#endif
-#include "debug_counter.h"
-#include "internal/sanitizers.h"
+#ifdef BUILDING_MODULAR_GC
+# define RB_DEBUG_COUNTER_INC(_name) ((void)0)
+# define RB_DEBUG_COUNTER_INC_IF(_name, cond) (!!(cond))
+#else
+# include "debug_counter.h"
+#endif
+
+#ifdef BUILDING_MODULAR_GC
+# define rb_asan_poison_object(obj) ((void)(obj))
+# define rb_asan_unpoison_object(obj, newobj_p) ((void)(obj), (void)(newobj_p))
+# define asan_unpoisoning_object(obj) if ((obj) || true)
+# define asan_poison_memory_region(ptr, size) ((void)(ptr), (void)(size))
+# define asan_unpoison_memory_region(ptr, size, malloc_p) ((void)(ptr), (size), (malloc_p))
+# define asan_unpoisoning_memory_region(ptr, size) if ((ptr) || (size) || true)
+
+# define VALGRIND_MAKE_MEM_DEFINED(ptr, size) ((void)(ptr), (void)(size))
+# define VALGRIND_MAKE_MEM_UNDEFINED(ptr, size) ((void)(ptr), (void)(size))
+#else
+# include "internal/sanitizers.h"
+#endif
/* MALLOC_HEADERS_BEGIN */
#ifndef HAVE_MALLOC_USABLE_SIZE
@@ -75,21 +97,12 @@
# include <mach/mach_port.h>
#endif
-#ifndef VM_CHECK_MODE
-# define VM_CHECK_MODE RUBY_DEBUG
-#endif
-
-// From ractor_core.h
-#ifndef RACTOR_CHECK_MODE
-# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE)
-#endif
-
#ifndef RUBY_DEBUG_LOG
# define RUBY_DEBUG_LOG(...)
#endif
-#ifndef GC_HEAP_INIT_SLOTS
-#define GC_HEAP_INIT_SLOTS 10000
+#ifndef GC_HEAP_INIT_BYTES
+#define GC_HEAP_INIT_BYTES (2560 * 1024)
#endif
#ifndef GC_HEAP_FREE_SLOTS
#define GC_HEAP_FREE_SLOTS 4096
@@ -97,8 +110,8 @@
#ifndef GC_HEAP_GROWTH_FACTOR
#define GC_HEAP_GROWTH_FACTOR 1.8
#endif
-#ifndef GC_HEAP_GROWTH_MAX_SLOTS
-#define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */
+#ifndef GC_HEAP_GROWTH_MAX_BYTES
+#define GC_HEAP_GROWTH_MAX_BYTES 0 /* 0 is disable */
#endif
#ifndef GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO
# define GC_HEAP_REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO 0.01
@@ -137,6 +150,17 @@
#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
#endif
+#ifndef GC_MALLOC_INCREASE_LOCAL_THRESHOLD
+#define GC_MALLOC_INCREASE_LOCAL_THRESHOLD (8 * 1024 /* 8KB */)
+#endif
+
+#ifdef RB_THREAD_LOCAL_SPECIFIER
+#define USE_MALLOC_INCREASE_LOCAL 1
+static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
+#else
+#define USE_MALLOC_INCREASE_LOCAL 0
+#endif
+
#ifndef GC_CAN_COMPILE_COMPACTION
#if defined(__wasi__) /* WebAssembly doesn't support signals */
# define GC_CAN_COMPILE_COMPACTION 0
@@ -155,12 +179,28 @@
#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
#ifndef HEAP_COUNT
-# define HEAP_COUNT 5
+# if SIZEOF_VALUE >= 8
+# define HEAP_COUNT 12
+# else
+# define HEAP_COUNT 5
+# endif
+#endif
+
+/* The reciprocal table and pool_slot_sizes array are both generated from this
+ * single definition, so they can never get out of sync. */
+#if SIZEOF_VALUE >= 8
+# define EACH_POOL_SLOT_SIZE(SLOT) \
+ SLOT(32) SLOT(40) SLOT(64) SLOT(80) SLOT(96) SLOT(128) \
+ SLOT(160) SLOT(256) SLOT(512) SLOT(640) SLOT(768) SLOT(1024)
+#else
+# define EACH_POOL_SLOT_SIZE(SLOT) \
+ SLOT(32) SLOT(64) SLOT(128) SLOT(256) SLOT(512)
#endif
typedef struct ractor_newobj_heap_cache {
struct free_slot *freelist;
struct heap_page *using_page;
+ size_t allocated_objects_count;
} rb_ractor_newobj_heap_cache_t;
typedef struct ractor_newobj_cache {
@@ -169,10 +209,10 @@ typedef struct ractor_newobj_cache {
} rb_ractor_newobj_cache_t;
typedef struct {
- size_t heap_init_slots[HEAP_COUNT];
+ size_t heap_init_bytes;
size_t heap_free_slots;
double growth_factor;
- size_t growth_max_slots;
+ size_t growth_max_bytes;
double heap_free_slots_min_ratio;
double heap_free_slots_goal_ratio;
@@ -190,10 +230,10 @@ typedef struct {
} ruby_gc_params_t;
static ruby_gc_params_t gc_params = {
- { GC_HEAP_INIT_SLOTS },
+ GC_HEAP_INIT_BYTES,
GC_HEAP_FREE_SLOTS,
GC_HEAP_GROWTH_FACTOR,
- GC_HEAP_GROWTH_MAX_SLOTS,
+ GC_HEAP_GROWTH_MAX_BYTES,
GC_HEAP_FREE_SLOTS_MIN_RATIO,
GC_HEAP_FREE_SLOTS_GOAL_RATIO,
@@ -268,9 +308,24 @@ int ruby_rgengc_debug;
#ifndef GC_ENABLE_LAZY_SWEEP
# define GC_ENABLE_LAZY_SWEEP 1
#endif
+
+#ifndef VERIFY_FREE_SIZE
+#if RUBY_DEBUG
+#define VERIFY_FREE_SIZE 1
+#else
+#define VERIFY_FREE_SIZE 0
+#endif
+#endif
+
+#if VERIFY_FREE_SIZE
+#undef CALC_EXACT_MALLOC_SIZE
+#define CALC_EXACT_MALLOC_SIZE 1
+#endif
+
#ifndef CALC_EXACT_MALLOC_SIZE
# define CALC_EXACT_MALLOC_SIZE 0
#endif
+
#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
# ifndef MALLOC_ALLOCATED_SIZE
# define MALLOC_ALLOCATED_SIZE 0
@@ -283,7 +338,7 @@ int ruby_rgengc_debug;
#endif
#ifndef GC_DEBUG_STRESS_TO_CLASS
-# define GC_DEBUG_STRESS_TO_CLASS 1
+# define GC_DEBUG_STRESS_TO_CLASS RUBY_DEBUG
#endif
typedef enum {
@@ -363,7 +418,6 @@ struct RMoved {
VALUE flags;
VALUE dummy;
VALUE destination;
- uint32_t original_shape_id;
};
#define RMOVED(obj) ((struct RMoved *)(obj))
@@ -424,7 +478,7 @@ typedef struct rb_heap_struct {
uintptr_t compact_cursor_index;
struct heap_page *pooled_pages;
size_t total_pages; /* total page count in a heap */
- size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */
+ size_t total_slots; /* total slot count */
} rb_heap_t;
@@ -442,10 +496,35 @@ enum gc_mode {
gc_mode_compacting,
};
+typedef rbimpl_atomic_uint64_t gc_counter_t;
+
+#if !defined(HAVE_GCC_ATOMIC_BUILTINS_64) && !defined(_WIN32) && \
+ !(defined(__sun) && defined(HAVE_ATOMIC_H) && (defined(_LP64) || defined(_I32LPx)))
+# define MALLOC_COUNTERS_NEED_LOCK 1
+#endif
+
+struct gc_malloc_bytes {
+ gc_counter_t malloc;
+ gc_counter_t free;
+
+ /* Snapshots of `malloc` / `free` taken at the end of the last GC */
+ gc_counter_t malloc_at_last_gc;
+ gc_counter_t free_at_last_gc;
+};
+
typedef struct rb_objspace {
struct {
+ struct gc_malloc_bytes counters;
+#if RGENGC_ESTIMATE_OLDMALLOC
+ struct gc_malloc_bytes oldcounters;
+#endif
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+ rb_nativethread_lock_t lock;
+#endif
+ } malloc_counters;
+
+ struct {
size_t limit;
- size_t increase;
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
size_t allocations;
@@ -465,14 +544,12 @@ typedef struct rb_objspace {
unsigned int during_compacting : 1;
unsigned int during_reference_updating : 1;
unsigned int gc_stressful: 1;
- unsigned int has_newobj_hook: 1;
unsigned int during_minor_gc : 1;
unsigned int during_incremental_marking : 1;
unsigned int measure_gc : 1;
} flags;
rb_event_flag_t hook_events;
- unsigned long long next_object_id;
rb_heap_t heaps[HEAP_COUNT];
size_t empty_pages_count;
@@ -493,7 +570,7 @@ typedef struct rb_objspace {
uintptr_t range[2];
size_t freeable_pages;
- size_t allocatable_slots;
+ size_t allocatable_bytes;
/* final */
VALUE deferred_final;
@@ -540,6 +617,7 @@ typedef struct rb_objspace {
double gc_sweep_start_time;
size_t total_allocated_objects_at_gc_start;
size_t heap_used_at_gc_start;
+ size_t heap_total_slots_at_gc_start;
/* basic statistics */
size_t count;
@@ -550,13 +628,14 @@ typedef struct rb_objspace {
/* Weak references */
size_t weak_references_count;
- size_t retained_weak_references_count;
} profile;
VALUE gc_stress_mode;
struct {
+ bool parent_object_old_p;
VALUE parent_object;
+
int need_major_gc;
size_t last_major_gc;
size_t uncollectible_wb_unprotected_objects;
@@ -565,7 +644,6 @@ typedef struct rb_objspace {
size_t old_objects_limit;
#if RGENGC_ESTIMATE_OLDMALLOC
- size_t oldmalloc_increase;
size_t oldmalloc_increase_limit;
#endif
@@ -591,17 +669,20 @@ typedef struct rb_objspace {
size_t step_slots;
} rincgc;
- st_table *id_to_obj_tbl;
- st_table *obj_to_id_tbl;
-
#if GC_DEBUG_STRESS_TO_CLASS
VALUE stress_to_class;
#endif
- rb_darray(VALUE *) weak_references;
+ rb_darray(VALUE) weak_references;
rb_postponed_job_handle_t finalize_deferred_pjob;
unsigned long live_ractor_cache_count;
+
+ int sweeping_heap_count;
+
+ int fork_vm_lock_lev;
+
+ struct rb_gc_vm_context vm_context;
} rb_objspace_t;
#ifndef HEAP_PAGE_ALIGN_LOG
@@ -609,10 +690,10 @@ typedef struct rb_objspace {
#define HEAP_PAGE_ALIGN_LOG 16
#endif
-#if RACTOR_CHECK_MODE || GC_DEBUG
+#if RB_GC_OBJ_HAS_SUFFIX || GC_DEBUG
struct rvalue_overhead {
-# if RACTOR_CHECK_MODE
- uint32_t _ractor_belonging_id;
+# if RB_GC_OBJ_HAS_SUFFIX
+ struct rb_gc_obj_suffix suffix;
# endif
# if GC_DEBUG
const char *file;
@@ -630,10 +711,37 @@ struct rvalue_overhead {
size_t rb_gc_impl_obj_slot_size(VALUE obj);
# define GET_RVALUE_OVERHEAD(obj) ((struct rvalue_overhead *)((uintptr_t)obj + rb_gc_impl_obj_slot_size(obj)))
#else
-# define RVALUE_OVERHEAD 0
+# ifndef RVALUE_OVERHEAD
+# define RVALUE_OVERHEAD 0
+# endif
#endif
-#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
+#define RVALUE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
+
+static const size_t pool_slot_sizes[HEAP_COUNT] = {
+#define SLOT(size) ((size) + RVALUE_OVERHEAD),
+ EACH_POOL_SLOT_SIZE(SLOT)
+#undef SLOT
+};
+
+/* Precomputed reciprocals for fast slot index calculation.
+ * For slot size d: reciprocal = ceil(2^48 / d).
+ * Then offset / d == (uint32_t)((offset * reciprocal) >> 48)
+ * for all offset < HEAP_PAGE_SIZE. */
+#define SLOT_RECIPROCAL_SHIFT 48
+#define SLOT_RECIPROCAL(size) (((1ULL << SLOT_RECIPROCAL_SHIFT) + (size) - 1) / (size))
+
+static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = {
+#define SLOT(size) SLOT_RECIPROCAL((size) + RVALUE_OVERHEAD),
+ EACH_POOL_SLOT_SIZE(SLOT)
+#undef SLOT
+};
+
+#if SIZEOF_VALUE >= 8
+static uint8_t size_to_heap_idx[1024 / 8 + 1];
+#else
+static uint8_t size_to_heap_idx[512 / 8 + 1];
+#endif
#ifndef MAX
# define MAX(a, b) (((a) > (b)) ? (a) : (b))
@@ -643,12 +751,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj);
#endif
#define roomof(x, y) (((x) + (y) - 1) / (y))
#define CEILDIV(i, mod) roomof(i, mod)
+#define MIN_POOL_SLOT_SIZE 32
enum {
HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
- HEAP_PAGE_OBJ_LIMIT = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header)) / BASE_SLOT_SIZE),
- HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, BASE_SLOT_SIZE), BITS_BITLENGTH),
+ HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, MIN_POOL_SLOT_SIZE), BITS_BITLENGTH),
HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
};
#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
@@ -710,6 +818,10 @@ struct free_slot {
};
struct heap_page {
+ /* Cache line 0: allocation fast path + SLOT_INDEX */
+ struct free_slot *freelist;
+ uintptr_t start;
+ uint64_t slot_size_reciprocal;
unsigned short slot_size;
unsigned short total_slots;
unsigned short free_slots;
@@ -725,8 +837,6 @@ struct heap_page {
struct heap_page *free_next;
struct heap_page_body *body;
- uintptr_t start;
- struct free_slot *freelist;
struct ccan_list_node page_node;
bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
@@ -787,44 +897,63 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
-#define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE)
-#define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
-#define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
-#define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
+static inline size_t
+slot_index_for_offset(size_t offset, uint64_t reciprocal)
+{
+ return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT);
+}
+
+#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal)
+#define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH)
+#define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
+#define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
-/* Bitmap Operations */
-#define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
-#define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
-#define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
+#define _MARKED_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] & SLOT_BITMAP_BIT(page, p))
+#define _MARK_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] |= SLOT_BITMAP_BIT(page, p))
+#define _CLEAR_IN_BITMAP(bits, page, p) ((bits)[SLOT_BITMAP_INDEX(page, p)] &= ~SLOT_BITMAP_BIT(page, p))
+
+#define MARKED_IN_BITMAP(bits, p) _MARKED_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
+#define MARK_IN_BITMAP(bits, p) _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
+#define CLEAR_IN_BITMAP(bits, p) _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
-/* getting bitmap */
#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
-#define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3
-
-#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT))
-#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT)
-
static int
RVALUE_AGE_GET(VALUE obj)
{
- bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
- return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK;
+ struct heap_page *page = GET_HEAP_PAGE(obj);
+ bits_t *age_bits = page->age_bits;
+ size_t slot_idx = SLOT_INDEX(page, obj);
+ size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
+ int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
+ int lo = (age_bits[idx] >> shift) & 1;
+ int hi = (age_bits[idx + 1] >> shift) & 1;
+ return lo | (hi << 1);
}
static void
-RVALUE_AGE_SET(VALUE obj, int age)
+RVALUE_AGE_SET_BITMAP(VALUE obj, int age)
{
RUBY_ASSERT(age <= RVALUE_OLD_AGE);
- bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits;
- // clear the bits
- age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj)));
- // shift the correct value in
- age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj));
+ struct heap_page *page = GET_HEAP_PAGE(obj);
+ bits_t *age_bits = page->age_bits;
+ size_t slot_idx = SLOT_INDEX(page, obj);
+ size_t idx = (slot_idx / BITS_BITLENGTH) * 2;
+ int shift = (int)(slot_idx & (BITS_BITLENGTH - 1));
+ bits_t mask = (bits_t)1 << shift;
+
+ age_bits[idx] = (age_bits[idx] & ~mask) | ((bits_t)(age & 1) << shift);
+ age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift);
+}
+
+static void
+RVALUE_AGE_SET(VALUE obj, int age)
+{
+ RVALUE_AGE_SET_BITMAP(obj, age);
if (age == RVALUE_OLD_AGE) {
RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED);
}
@@ -834,8 +963,112 @@ RVALUE_AGE_SET(VALUE obj, int age)
}
#define malloc_limit objspace->malloc_params.limit
-#define malloc_increase objspace->malloc_params.increase
+#define malloc_increase gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters)
#define malloc_allocated_size objspace->malloc_params.allocated_size
+
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+# define MALLOC_COUNTERS_LOCK(o) rb_native_mutex_lock(&(o)->malloc_counters.lock)
+# define MALLOC_COUNTERS_UNLOCK(o) rb_native_mutex_unlock(&(o)->malloc_counters.lock)
+#else
+# define MALLOC_COUNTERS_LOCK(o) ((void)0)
+# define MALLOC_COUNTERS_UNLOCK(o) ((void)0)
+#endif
+
+static inline void
+gc_counter_add(gc_counter_t *p, size_t delta)
+{
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+ *p += (gc_counter_t)delta;
+#else
+ rbimpl_atomic_u64_fetch_add_relaxed(p, (uint64_t)delta);
+#endif
+}
+
+static inline gc_counter_t
+gc_counter_load_relaxed(const gc_counter_t *p)
+{
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+ return *p;
+#else
+ return rbimpl_atomic_u64_load_relaxed(p);
+#endif
+}
+
+static inline gc_counter_t
+gc_counter_load_acquire(const gc_counter_t *p)
+{
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+ return *p;
+#else
+ return rbimpl_atomic_u64_load_acquire(p);
+#endif
+}
+
+static inline void
+gc_counter_store_release(gc_counter_t *p, gc_counter_t v)
+{
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+ *p = v;
+#else
+ rbimpl_atomic_u64_set_release(p, v);
+#endif
+}
+
+static inline int64_t
+gc_malloc_counters_increase(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
+{
+ MALLOC_COUNTERS_LOCK(objspace);
+ gc_counter_t malloc_at = gc_counter_load_acquire(&c->malloc_at_last_gc);
+ gc_counter_t free_at = gc_counter_load_acquire(&c->free_at_last_gc);
+ gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
+ gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
+ MALLOC_COUNTERS_UNLOCK(objspace);
+
+ gc_counter_t malloc_delta = malloc_now - malloc_at;
+ gc_counter_t free_delta = free_now - free_at;
+
+ if (malloc_delta >= free_delta) {
+ return (int64_t)(malloc_delta - free_delta);
+ }
+ else {
+ return -(int64_t)(free_delta - malloc_delta);
+ }
+}
+
+static inline size_t
+gc_malloc_counters_increase_unsigned(rb_objspace_t *objspace, const struct gc_malloc_bytes *c)
+{
+ int64_t inc = gc_malloc_counters_increase(objspace, c);
+ if (inc <= 0) return 0;
+#if SIZEOF_SIZE_T < 8
+ if ((uint64_t)inc > SIZE_MAX) return SIZE_MAX;
+#endif
+ return (size_t)inc;
+}
+
+static inline int64_t
+gc_malloc_counters_snapshot(rb_objspace_t *objspace, struct gc_malloc_bytes *c)
+{
+ MALLOC_COUNTERS_LOCK(objspace);
+ gc_counter_t malloc_now = gc_counter_load_relaxed(&c->malloc);
+ gc_counter_t free_now = gc_counter_load_relaxed(&c->free);
+ gc_counter_t malloc_at = gc_counter_load_relaxed(&c->malloc_at_last_gc);
+ gc_counter_t free_at = gc_counter_load_relaxed(&c->free_at_last_gc);
+ gc_counter_store_release(&c->malloc_at_last_gc, malloc_now);
+ gc_counter_store_release(&c->free_at_last_gc, free_now);
+ MALLOC_COUNTERS_UNLOCK(objspace);
+
+ gc_counter_t malloc_delta = malloc_now - malloc_at;
+ gc_counter_t free_delta = free_now - free_at;
+
+ if (malloc_delta >= free_delta) {
+ return (int64_t)(malloc_delta - free_delta);
+ }
+ else {
+ return -(int64_t)(free_delta - malloc_delta);
+ }
+}
+
#define heap_pages_lomem objspace->heap_pages.range[0]
#define heap_pages_himem objspace->heap_pages.range[1]
#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
@@ -901,12 +1134,7 @@ gc_mode_verify(enum gc_mode mode)
static inline bool
has_sweeping_pages(rb_objspace_t *objspace)
{
- for (int i = 0; i < HEAP_COUNT; i++) {
- if ((&heaps[i])->sweeping_page) {
- return TRUE;
- }
- }
- return FALSE;
+ return objspace->sweeping_heap_count != 0;
}
static inline size_t
@@ -961,9 +1189,22 @@ total_final_slots_count(rb_objspace_t *objspace)
#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
-#define GC_INCREMENTAL_SWEEP_SLOT_COUNT 2048
-#define GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT 1024
+/*
+ * Byte budget for incremental sweep steps. Each step sweeps at most
+ * this many bytes worth of slots before yielding. The effective slot
+ * count per step is GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size,
+ * so larger slot pools (which are less heavily used) naturally get
+ * fewer slots swept per step.
+ *
+ * Baseline: 2048 slots * RVALUE_SLOT_SIZE = 2048 * 40 = 81920 bytes,
+ * preserving the historical behavior for the smallest heap.
+ */
+#define GC_INCREMENTAL_SWEEP_BYTES (2048 * RVALUE_SLOT_SIZE)
+#define GC_INCREMENTAL_SWEEP_POOL_BYTES (1024 * RVALUE_SLOT_SIZE)
#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
+/* In lazy sweeping or the previous incremental marking finished and did not yield a free page. */
+#define needs_continue_sweeping(objspace, heap) \
+ ((heap)->free_pages == NULL && is_lazy_sweeping(objspace))
#if SIZEOF_LONG == SIZEOF_VOIDP
# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
@@ -975,7 +1216,7 @@ total_final_slots_count(rb_objspace_t *objspace)
#endif
struct RZombie {
- struct RBasic basic;
+ VALUE flags;
VALUE next;
void (*dfree)(void *);
void *data;
@@ -983,9 +1224,9 @@ struct RZombie {
#define RZOMBIE(o) ((struct RZombie *)(o))
-int ruby_enable_autocompact = 0;
+static bool ruby_enable_autocompact = false;
#if RGENGC_CHECK_MODE
-gc_compact_compare_func ruby_autocompact_compare_func;
+static gc_compact_compare_func ruby_autocompact_compare_func;
#endif
static void init_mark_stack(mark_stack_t *stack);
@@ -1089,10 +1330,10 @@ tick(void)
return val;
}
+#elif defined(__POWERPC__) && defined(__APPLE__)
/* Implementation for macOS PPC by @nobu
* See: https://github.com/ruby/ruby/pull/5975#discussion_r890045558
*/
-#elif defined(__POWERPC__) && defined(__APPLE__)
typedef unsigned long long tick_t;
#define PRItick "llu"
@@ -1211,7 +1452,7 @@ check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int ter
{
int err = 0;
- int lev = rb_gc_vm_lock_no_barrier();
+ int lev = RB_GC_VM_LOCK_NO_BARRIER();
{
if (SPECIAL_CONST_P(obj)) {
fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
@@ -1301,7 +1542,7 @@ check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int ter
}
}
}
- rb_gc_vm_unlock_no_barrier(lev);
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
if (err > 0 && terminate) {
rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
@@ -1327,16 +1568,12 @@ check_rvalue_consistency(rb_objspace_t *objspace, const VALUE obj)
static inline bool
gc_object_moved_p(rb_objspace_t *objspace, VALUE obj)
{
- if (RB_SPECIAL_CONST_P(obj)) {
- return FALSE;
- }
- else {
- int ret;
- asan_unpoisoning_object(obj) {
- ret = BUILTIN_TYPE(obj) == T_MOVED;
- }
- return ret;
+
+ bool ret;
+ asan_unpoisoning_object(obj) {
+ ret = BUILTIN_TYPE(obj) == T_MOVED;
}
+ return ret;
}
static inline int
@@ -1477,7 +1714,6 @@ rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event)
{
rb_objspace_t *objspace = objspace_ptr;
objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
- objspace->flags.has_newobj_hook = !!(objspace->hook_events & RUBY_INTERNAL_EVENT_NEWOBJ);
}
unsigned long long
@@ -1507,38 +1743,6 @@ rb_gc_impl_get_measure_total_time(void *objspace_ptr)
return objspace->flags.measure_gc;
}
-static size_t
-minimum_slots_for_heap(rb_objspace_t *objspace, rb_heap_t *heap)
-{
- size_t heap_idx = heap - heaps;
- return gc_params.heap_init_slots[heap_idx];
-}
-
-static int
-object_id_cmp(st_data_t x, st_data_t y)
-{
- if (RB_TYPE_P(x, T_BIGNUM)) {
- return !rb_big_eql(x, y);
- }
- else {
- return x != y;
- }
-}
-
-static st_index_t
-object_id_hash(st_data_t n)
-{
- return FIX2LONG(rb_hash((VALUE)n));
-}
-
-#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
-#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT)
-
-static const struct st_hash_type object_id_hash_type = {
- object_id_cmp,
- object_id_hash,
-};
-
/* garbage objects will be collected soon. */
bool
rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr)
@@ -1564,54 +1768,12 @@ rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE ptr)
!RVALUE_MARKED(objspace, ptr);
}
-VALUE
-rb_gc_impl_object_id_to_ref(void *objspace_ptr, VALUE object_id)
+struct rb_gc_vm_context *
+rb_gc_impl_get_vm_context(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
- VALUE obj;
- if (st_lookup(objspace->id_to_obj_tbl, object_id, &obj) &&
- !rb_gc_impl_garbage_object_p(objspace, obj)) {
- return obj;
- }
-
- if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(objspace->next_object_id))) {
- rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
- }
- else {
- rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
- }
-}
-
-VALUE
-rb_gc_impl_object_id(void *objspace_ptr, VALUE obj)
-{
- VALUE id;
- rb_objspace_t *objspace = objspace_ptr;
-
- unsigned int lev = rb_gc_vm_lock();
- if (FL_TEST(obj, FL_SEEN_OBJ_ID)) {
- st_data_t val;
- if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &val)) {
- id = (VALUE)val;
- }
- else {
- rb_bug("rb_gc_impl_object_id: FL_SEEN_OBJ_ID flag set but not found in table");
- }
- }
- else {
- GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, NULL));
-
- id = ULL2NUM(objspace->next_object_id);
- objspace->next_object_id += OBJ_ID_INCREMENT;
-
- st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
- st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
- FL_SET(obj, FL_SEEN_OBJ_ID);
- }
- rb_gc_vm_unlock(lev);
-
- return id;
+ return &objspace->vm_context;
}
static void free_stack_chunks(mark_stack_t *);
@@ -1631,13 +1793,14 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj
page->freelist = slot;
asan_lock_freelist(page);
- RVALUE_AGE_RESET(obj);
+ // Should have already been reset
+ GC_ASSERT(RVALUE_AGE_GET(obj) == 0);
if (RGENGC_CHECK_MODE &&
/* obj should belong to page */
!(page->start <= (uintptr_t)obj &&
(uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
- obj % BASE_SLOT_SIZE == 0)) {
+ obj % sizeof(VALUE) == 0)) {
rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)obj);
}
@@ -1646,8 +1809,8 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj
}
static void
-heap_allocatable_slots_expand(rb_objspace_t *objspace,
- rb_heap_t *heap, size_t free_slots, size_t total_slots)
+heap_allocatable_bytes_expand(rb_objspace_t *objspace,
+ rb_heap_t *heap, size_t free_slots, size_t total_slots, size_t slot_size)
{
double goal_ratio = gc_params.heap_free_slots_goal_ratio;
size_t target_total_slots;
@@ -1656,7 +1819,7 @@ heap_allocatable_slots_expand(rb_objspace_t *objspace,
target_total_slots = (size_t)(total_slots * gc_params.growth_factor);
}
else if (total_slots == 0) {
- target_total_slots = minimum_slots_for_heap(objspace, heap);
+ target_total_slots = gc_params.heap_init_bytes / slot_size;
}
else {
/* Find `f' where free_slots = f * total_slots * goal_ratio
@@ -1679,8 +1842,8 @@ heap_allocatable_slots_expand(rb_objspace_t *objspace,
}
}
- if (gc_params.growth_max_slots > 0) {
- size_t max_total_slots = (size_t)(total_slots + gc_params.growth_max_slots);
+ if (gc_params.growth_max_bytes > 0) {
+ size_t max_total_slots = total_slots + gc_params.growth_max_bytes / slot_size;
if (target_total_slots > max_total_slots) target_total_slots = max_total_slots;
}
@@ -1688,7 +1851,7 @@ heap_allocatable_slots_expand(rb_objspace_t *objspace,
/* Extend by at least 1 page. */
if (extend_slot_count == 0) extend_slot_count = 1;
- objspace->heap_pages.allocatable_slots += extend_slot_count;
+ objspace->heap_pages.allocatable_bytes += extend_slot_count * slot_size;
}
static inline void
@@ -1771,13 +1934,7 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
static void
heap_pages_free_unused_pages(rb_objspace_t *objspace)
{
- size_t pages_to_keep_count =
- // Get number of pages estimated for the smallest size pool
- CEILDIV(objspace->heap_pages.allocatable_slots, HEAP_PAGE_OBJ_LIMIT) *
- // Estimate the average slot size multiple
- (1 << (HEAP_COUNT / 2));
-
- if (objspace->empty_pages != NULL && objspace->empty_pages_count > pages_to_keep_count) {
+ if (objspace->empty_pages != NULL && heap_pages_freeable_pages > 0) {
GC_ASSERT(objspace->empty_pages_count > 0);
objspace->empty_pages = NULL;
objspace->empty_pages_count = 0;
@@ -1786,15 +1943,15 @@ heap_pages_free_unused_pages(rb_objspace_t *objspace)
for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
- if (heap_page_in_global_empty_pages_pool(objspace, page) && pages_to_keep_count == 0) {
+ if (heap_page_in_global_empty_pages_pool(objspace, page) && heap_pages_freeable_pages > 0) {
heap_page_free(objspace, page);
+ heap_pages_freeable_pages--;
}
else {
- if (heap_page_in_global_empty_pages_pool(objspace, page) && pages_to_keep_count > 0) {
+ if (heap_page_in_global_empty_pages_pool(objspace, page)) {
page->free_next = objspace->empty_pages;
objspace->empty_pages = page;
objspace->empty_pages_count++;
- pages_to_keep_count--;
}
if (i != j) {
@@ -1916,7 +2073,10 @@ static struct heap_page *
heap_page_resurrect(rb_objspace_t *objspace)
{
struct heap_page *page = NULL;
- if (objspace->empty_pages != NULL) {
+ if (objspace->empty_pages == NULL) {
+ GC_ASSERT(objspace->empty_pages_count == 0);
+ }
+ else {
GC_ASSERT(objspace->empty_pages_count > 0);
objspace->empty_pages_count--;
page = objspace->empty_pages;
@@ -1981,31 +2141,22 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
GC_ASSERT(!heap->sweeping_page);
GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));
- /* adjust obj_limit (object number available in this page) */
+ /* Align start to slot_size boundary */
uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
- if (start % BASE_SLOT_SIZE != 0) {
- int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE);
- start = start + delta;
- GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1);
-
- /* Find a num in page that is evenly divisible by `stride`.
- * This is to ensure that objects are aligned with bit planes.
- * In other words, ensure there are an even number of objects
- * per bit plane. */
- if (NUM_IN_PAGE(start) == 1) {
- start += heap->slot_size - BASE_SLOT_SIZE;
- }
-
- GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % heap->slot_size == 0);
- }
+ uintptr_t rem = start % heap->slot_size;
+ if (rem) start += heap->slot_size - rem;
int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
page->start = start;
page->total_slots = slot_count;
page->slot_size = heap->slot_size;
+ page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps];
page->heap = heap;
+ memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
+ memset(&page->age_bits[0], 0, sizeof(page->age_bits));
+
asan_unlock_freelist(page);
page->freelist = NULL;
asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false);
@@ -2026,47 +2177,56 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
static int
heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_heap_t *heap)
{
- if (objspace->heap_pages.allocatable_slots > 0) {
- gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", "
- "allocatable_slots: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
- rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_slots, heap->total_pages);
+ gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", "
+ "allocatable_bytes: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
+ rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_bytes, heap->total_pages);
- struct heap_page *page = heap_page_resurrect(objspace);
- if (page == NULL) {
- page = heap_page_allocate(objspace);
- }
+ bool allocated = false;
+ struct heap_page *page = heap_page_resurrect(objspace);
+
+ if (page == NULL && objspace->heap_pages.allocatable_bytes > 0) {
+ page = heap_page_allocate(objspace);
+ allocated = true;
+
+ GC_ASSERT(page != NULL);
+ }
+
+ if (page != NULL) {
heap_add_page(objspace, heap, page);
heap_add_freepage(heap, page);
- if (objspace->heap_pages.allocatable_slots > (size_t)page->total_slots) {
- objspace->heap_pages.allocatable_slots -= page->total_slots;
- }
- else {
- objspace->heap_pages.allocatable_slots = 0;
+ if (allocated) {
+ size_t page_bytes = (size_t)page->total_slots * page->slot_size;
+ if (objspace->heap_pages.allocatable_bytes > page_bytes) {
+ objspace->heap_pages.allocatable_bytes -= page_bytes;
+ }
+ else {
+ objspace->heap_pages.allocatable_bytes = 0;
+ }
}
-
- return true;
}
- return false;
+ return page != NULL;
}
static void
heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_heap_t *heap)
{
- size_t prev_allocatable_slots = objspace->heap_pages.allocatable_slots;
- // Set allocatable slots to 1 to force a page to be created.
- objspace->heap_pages.allocatable_slots = 1;
+ size_t prev_allocatable_bytes = objspace->heap_pages.allocatable_bytes;
+ objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
heap_page_allocate_and_initialize(objspace, heap);
GC_ASSERT(heap->free_pages != NULL);
- objspace->heap_pages.allocatable_slots = prev_allocatable_slots;
+ objspace->heap_pages.allocatable_bytes = prev_allocatable_bytes;
}
static void
gc_continue(rb_objspace_t *objspace, rb_heap_t *heap)
{
unsigned int lock_lev;
- gc_enter(objspace, gc_enter_event_continue, &lock_lev);
+ bool needs_gc = is_incremental_marking(objspace) || needs_continue_sweeping(objspace, heap);
+ if (!needs_gc) return;
+
+ gc_enter(objspace, gc_enter_event_continue, &lock_lev); // takes vm barrier, try to avoid
/* Continue marking if in incremental marking. */
if (is_incremental_marking(objspace)) {
@@ -2075,9 +2235,7 @@ gc_continue(rb_objspace_t *objspace, rb_heap_t *heap)
}
}
- /* Continue sweeping if in lazy sweeping or the previous incremental
- * marking finished and did not yield a free page. */
- if (heap->free_pages == NULL && is_lazy_sweeping(objspace)) {
+ if (needs_continue_sweeping(objspace, heap)) {
gc_sweep_continue(objspace, heap);
}
@@ -2089,7 +2247,7 @@ heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
{
GC_ASSERT(heap->free_pages == NULL);
- if (heap->total_slots < gc_params.heap_init_slots[heap - heaps] &&
+ if (heap->total_slots < gc_params.heap_init_bytes / heap->slot_size &&
heap->sweeping_page == NULL) {
heap_page_allocate_and_initialize_force(objspace, heap);
GC_ASSERT(heap->free_pages != NULL);
@@ -2106,15 +2264,18 @@ heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
/* If we still don't have a free page and not allowed to create a new page,
* we should start a new GC cycle. */
if (heap->free_pages == NULL) {
+ GC_ASSERT(objspace->empty_pages_count == 0);
+ GC_ASSERT(objspace->heap_pages.allocatable_bytes == 0);
+
if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
rb_memerror();
}
else {
- if (objspace->heap_pages.allocatable_slots == 0 && !gc_config_full_mark_val) {
- heap_allocatable_slots_expand(objspace, heap,
+ if (objspace->heap_pages.allocatable_bytes == 0 && !gc_config_full_mark_val) {
+ heap_allocatable_bytes_expand(objspace, heap,
heap->freed_slots + heap->empty_slots,
- heap->total_slots);
- GC_ASSERT(objspace->heap_pages.allocatable_slots > 0);
+ heap->total_slots, heap->slot_size);
+ GC_ASSERT(objspace->heap_pages.allocatable_bytes > 0);
}
/* Do steps of incremental marking or lazy sweeping if the GC run permits. */
gc_continue(objspace, heap);
@@ -2147,16 +2308,6 @@ heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
GC_ASSERT(heap->free_pages != NULL);
}
-static inline VALUE
-newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3)
-{
- VALUE *p = (VALUE *)obj;
- p[2] = v1;
- p[3] = v2;
- p[4] = v3;
- return obj;
-}
-
#if GC_DEBUG
static inline const char*
rb_gc_impl_source_location_cstr(int *ptr)
@@ -2178,27 +2329,16 @@ rb_gc_impl_source_location_cstr(int *ptr)
static inline VALUE
newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
{
-#if !__has_feature(memory_sanitizer)
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
-#endif
RBASIC(obj)->flags = flags;
*((VALUE *)&RBASIC(obj)->klass) = klass;
-
- int t = flags & RUBY_T_MASK;
- if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
- RVALUE_AGE_SET_CANDIDATE(objspace, obj);
- }
-
-#if RACTOR_CHECK_MODE
- void rb_ractor_setup_belonging(VALUE obj);
- rb_ractor_setup_belonging(obj);
+#if RBASIC_SHAPE_ID_FIELD
+ RBASIC(obj)->shape_id = 0;
#endif
#if RGENGC_CHECK_MODE
- newobj_fill(obj, 0, 0, 0);
-
- int lev = rb_gc_vm_lock_no_barrier();
+ int lev = RB_GC_VM_LOCK_NO_BARRIER();
{
check_rvalue_consistency(objspace, obj);
@@ -2209,7 +2349,7 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
if (RVALUE_REMEMBERED(objspace, obj)) rb_bug("newobj: %s is remembered.", rb_obj_info(obj));
}
- rb_gc_vm_unlock_no_barrier(lev);
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
#endif
if (RB_UNLIKELY(wb_protected == FALSE)) {
@@ -2238,7 +2378,7 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
gc_report(5, objspace, "newobj: %s\n", rb_obj_info(obj));
- RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj));
+ // RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, rb_obj_info(obj));
return obj;
}
@@ -2253,22 +2393,26 @@ heap_slot_size(unsigned char pool_id)
{
GC_ASSERT(pool_id < HEAP_COUNT);
- size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
-
-#if RGENGC_CHECK_MODE
- rb_objspace_t *objspace = rb_gc_get_objspace();
- GC_ASSERT(heaps[pool_id].slot_size == (short)slot_size);
-#endif
-
- slot_size -= RVALUE_OVERHEAD;
-
- return slot_size;
+ return pool_slot_sizes[pool_id] - RVALUE_OVERHEAD;
}
bool
rb_gc_impl_size_allocatable_p(size_t size)
{
- return size <= heap_slot_size(HEAP_COUNT - 1);
+ return size + RVALUE_OVERHEAD <= pool_slot_sizes[HEAP_COUNT - 1];
+}
+
+static const size_t ALLOCATED_COUNT_STEP = 1024;
+static void
+ractor_cache_flush_count(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache)
+{
+ for (int heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
+ rb_ractor_newobj_heap_cache_t *heap_cache = &cache->heap_caches[heap_idx];
+
+ rb_heap_t *heap = &heaps[heap_idx];
+ RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, heap_cache->allocated_objects_count);
+ heap_cache->allocated_objects_count = 0;
+ }
}
static inline VALUE
@@ -2293,6 +2437,14 @@ ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *ca
VALUE obj = (VALUE)p;
rb_asan_unpoison_object(obj, true);
heap_cache->freelist = p->next;
+
+ heap_cache->allocated_objects_count++;
+ rb_heap_t *heap = &heaps[heap_idx];
+ if (heap_cache->allocated_objects_count >= ALLOCATED_COUNT_STEP) {
+ RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, heap_cache->allocated_objects_count);
+ heap_cache->allocated_objects_count = 0;
+ }
+
#if RGENGC_CHECK_MODE
GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == heap_slot_size(heap_idx));
// zero clear
@@ -2346,28 +2498,30 @@ ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache,
rb_asan_poison_object((VALUE)heap_cache->freelist);
}
+static void
+init_size_to_heap_idx(void)
+{
+ for (size_t i = 0; i < sizeof(size_to_heap_idx); i++) {
+ size_t effective = i * 8 + RVALUE_OVERHEAD;
+ uint8_t idx;
+ for (idx = 0; idx < HEAP_COUNT; idx++) {
+ if (effective <= pool_slot_sizes[idx]) break;
+ }
+ size_to_heap_idx[i] = idx;
+ }
+}
+
static inline size_t
heap_idx_for_size(size_t size)
{
- size += RVALUE_OVERHEAD;
-
- size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
-
- /* heap_idx is ceil(log2(slot_count)) */
- size_t heap_idx = 64 - nlz_int64(slot_count - 1);
-
- if (heap_idx >= HEAP_COUNT) {
- rb_bug("heap_idx_for_size: allocation size too large "
- "(size=%"PRIuSIZE"u, heap_idx=%"PRIuSIZE"u)", size, heap_idx);
+ size_t compressed = (size + 7) >> 3;
+ if (compressed < sizeof(size_to_heap_idx)) {
+ size_t heap_idx = size_to_heap_idx[compressed];
+ if (RB_LIKELY(heap_idx < HEAP_COUNT)) return heap_idx;
}
-#if RGENGC_CHECK_MODE
- rb_objspace_t *objspace = rb_gc_get_objspace();
- GC_ASSERT(size <= (size_t)heaps[heap_idx].slot_size);
- if (heap_idx > 0) GC_ASSERT(size > (size_t)heaps[heap_idx - 1].slot_size);
-#endif
-
- return heap_idx;
+ rb_bug("heap_idx_for_size: allocation size too large "
+ "(size=%"PRIuSIZE")", size);
}
size_t
@@ -2403,7 +2557,7 @@ newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size
bool unlock_vm = false;
if (!vm_locked) {
- lev = rb_gc_cr_lock();
+ lev = RB_GC_CR_LOCK();
unlock_vm = true;
}
@@ -2427,7 +2581,7 @@ newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size
}
if (unlock_vm) {
- rb_gc_cr_unlock(lev);
+ RB_GC_CR_UNLOCK(lev);
}
if (RB_UNLIKELY(obj == Qfalse)) {
@@ -2445,12 +2599,6 @@ newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t he
obj = newobj_cache_miss(objspace, cache, heap_idx, vm_locked);
}
- rb_heap_t *heap = &heaps[heap_idx];
- heap->total_allocated_objects++;
- GC_ASSERT(rb_gc_multi_ractor_p() ||
- heap->total_slots >=
- (heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count));
-
return obj;
}
@@ -2462,7 +2610,7 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_new
VALUE obj;
unsigned int lev;
- lev = rb_gc_cr_lock();
+ lev = RB_GC_CR_LOCK();
{
if (RB_UNLIKELY(during_gc || ruby_gc_stressful)) {
if (during_gc) {
@@ -2484,7 +2632,7 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_new
obj = newobj_alloc(objspace, cache, heap_idx, true);
newobj_init(klass, flags, wb_protected, objspace, obj);
}
- rb_gc_cr_unlock(lev);
+ RB_GC_CR_UNLOCK(lev);
return obj;
}
@@ -2507,7 +2655,7 @@ newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace
}
VALUE
-rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size)
+rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
{
VALUE obj;
rb_objspace_t *objspace = objspace_ptr;
@@ -2516,7 +2664,7 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
(void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
if (RB_UNLIKELY(stress_to_class)) {
- if (RTEST(rb_hash_has_key(stress_to_class, klass))) {
+ if (rb_hash_lookup2(stress_to_class, klass, Qundef) != Qundef) {
rb_memerror();
}
}
@@ -2538,7 +2686,7 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
newobj_slowpath_wb_unprotected(klass, flags, objspace, cache, heap_idx);
}
- return newobj_fill(obj, v1, v2, v3);
+ return obj;
}
static int
@@ -2590,7 +2738,7 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_range);
- if (p % BASE_SLOT_SIZE != 0) return FALSE;
+ if (p % sizeof(VALUE) != 0) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_align);
page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
@@ -2602,7 +2750,7 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
else {
if (p < page->start) return FALSE;
if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
- if ((NUM_IN_PAGE(p) * BASE_SLOT_SIZE) % page->slot_size != 0) return FALSE;
+ if ((p - page->start) % page->slot_size != 0) return FALSE;
return TRUE;
}
@@ -2616,7 +2764,7 @@ rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr)
return is_pointer_to_heap(objspace_ptr, ptr);
}
-#define ZOMBIE_OBJ_KEPT_FLAGS (FL_SEEN_OBJ_ID | FL_FINALIZE)
+#define ZOMBIE_OBJ_KEPT_FLAGS (FL_FINALIZE)
void
rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data)
@@ -2624,7 +2772,7 @@ rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), voi
rb_objspace_t *objspace = objspace_ptr;
struct RZombie *zombie = RZOMBIE(obj);
- zombie->basic.flags = T_ZOMBIE | (zombie->basic.flags & ZOMBIE_OBJ_KEPT_FLAGS);
+ zombie->flags = T_ZOMBIE | (zombie->flags & ZOMBIE_OBJ_KEPT_FLAGS);
zombie->dfree = dfree;
zombie->data = data;
VALUE prev, next = heap_pages_deferred_final;
@@ -2638,23 +2786,6 @@ rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), voi
page->heap->final_slots_count++;
}
-static void
-obj_free_object_id(rb_objspace_t *objspace, VALUE obj)
-{
- st_data_t o = (st_data_t)obj, id;
-
- GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE || FL_TEST(obj, FL_SEEN_OBJ_ID));
- FL_UNSET(obj, FL_SEEN_OBJ_ID);
-
- if (st_delete(objspace->obj_to_id_tbl, &o, &id)) {
- GC_ASSERT(id);
- st_delete(objspace->id_to_obj_tbl, &id, NULL);
- }
- else {
- rb_bug("Object ID seen, but not in mapping table: %s", rb_obj_info(obj));
- }
-}
-
typedef int each_obj_callback(void *, void *, size_t, void *);
typedef int each_page_callback(struct heap_page *, void *);
@@ -2816,30 +2947,38 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
RBASIC(obj)->flags |= FL_FINALIZE;
+ unsigned int lev = RB_GC_VM_LOCK();
+
if (st_lookup(finalizer_table, obj, &data)) {
table = (VALUE)data;
+ VALUE dup_table = rb_ary_dup(table);
+ RB_GC_VM_UNLOCK(lev);
/* avoid duplicate block, table is usually small */
{
long len = RARRAY_LEN(table);
long i;
for (i = 0; i < len; i++) {
- VALUE recv = RARRAY_AREF(table, i);
- if (rb_equal(recv, block)) {
+ VALUE recv = RARRAY_AREF(dup_table, i);
+ if (rb_equal(recv, block)) { // can't be called with VM lock held
return recv;
}
}
}
+ lev = RB_GC_VM_LOCK();
+ RB_GC_GUARD(dup_table);
rb_ary_push(table, block);
}
else {
- table = rb_ary_new3(1, block);
+ table = rb_ary_new3(2, rb_obj_id(obj), block);
rb_obj_hide(table);
st_add_direct(finalizer_table, obj, table);
}
+ RB_GC_VM_UNLOCK(lev);
+
return block;
}
@@ -2851,7 +2990,11 @@ rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
GC_ASSERT(!OBJ_FROZEN(obj));
st_data_t data = obj;
+
+ int lev = RB_GC_VM_LOCK();
st_delete(finalizer_table, &data, 0);
+ RB_GC_VM_UNLOCK(lev);
+
FL_UNSET(obj, FL_FINALIZE);
}
@@ -2864,27 +3007,17 @@ rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
if (!FL_TEST(obj, FL_FINALIZE)) return;
+ int lev = RB_GC_VM_LOCK();
if (RB_LIKELY(st_lookup(finalizer_table, obj, &data))) {
- table = (VALUE)data;
+ table = rb_ary_dup((VALUE)data);
+ RARRAY_ASET(table, 0, rb_obj_id(dest));
st_insert(finalizer_table, dest, table);
FL_SET(dest, FL_FINALIZE);
}
else {
rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
}
-}
-
-static VALUE
-get_object_id_in_finalizer(rb_objspace_t *objspace, VALUE obj)
-{
- if (FL_TEST(obj, FL_SEEN_OBJ_ID)) {
- return rb_gc_impl_object_id(objspace, obj);
- }
- else {
- VALUE id = ULL2NUM(objspace->next_object_id);
- objspace->next_object_id += OBJ_ID_INCREMENT;
- return id;
- }
+ RB_GC_VM_UNLOCK(lev);
}
static VALUE
@@ -2892,11 +3025,11 @@ get_final(long i, void *data)
{
VALUE table = (VALUE)data;
- return RARRAY_AREF(table, i);
+ return RARRAY_AREF(table, i + 1);
}
-static void
-run_final(rb_objspace_t *objspace, VALUE zombie)
+static unsigned int
+run_final(rb_objspace_t *objspace, VALUE zombie, unsigned int lev)
{
if (RZOMBIE(zombie)->dfree) {
RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
@@ -2907,7 +3040,9 @@ run_final(rb_objspace_t *objspace, VALUE zombie)
FL_UNSET(zombie, FL_FINALIZE);
st_data_t table;
if (st_delete(finalizer_table, &key, &table)) {
- rb_gc_run_obj_finalizer(get_object_id_in_finalizer(objspace, zombie), RARRAY_LEN(table), get_final, (void *)table);
+ RB_GC_VM_UNLOCK(lev);
+ rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
+ lev = RB_GC_VM_LOCK();
}
else {
rb_bug("FL_FINALIZE flag is set, but finalizers are not found");
@@ -2916,6 +3051,7 @@ run_final(rb_objspace_t *objspace, VALUE zombie)
else {
GC_ASSERT(!st_lookup(finalizer_table, key, NULL));
}
+ return lev;
}
static void
@@ -2928,25 +3064,22 @@ finalize_list(rb_objspace_t *objspace, VALUE zombie)
next_zombie = RZOMBIE(zombie)->next;
page = GET_HEAP_PAGE(zombie);
- run_final(objspace, zombie);
+ unsigned int lev = RB_GC_VM_LOCK();
- int lev = rb_gc_vm_lock();
+ lev = run_final(objspace, zombie, lev);
{
GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
- if (FL_TEST(zombie, FL_SEEN_OBJ_ID)) {
- obj_free_object_id(objspace, zombie);
- }
-
GC_ASSERT(page->heap->final_slots_count > 0);
GC_ASSERT(page->final_slots > 0);
page->heap->final_slots_count--;
page->final_slots--;
page->free_slots++;
+ RVALUE_AGE_SET_BITMAP(zombie, 0);
heap_page_add_freeobj(objspace, page, zombie);
page->heap->total_freed_objects++;
}
- rb_gc_vm_unlock(lev);
+ RB_GC_VM_UNLOCK(lev);
zombie = next_zombie;
}
@@ -3002,6 +3135,7 @@ gc_abort(void *objspace_ptr)
}
if (is_lazy_sweeping(objspace)) {
+ objspace->sweeping_heap_count = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
@@ -3048,16 +3182,15 @@ rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
}
static int
-rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t data)
+rb_gc_impl_shutdown_call_finalizer_i(st_data_t key, st_data_t val, st_data_t _data)
{
- rb_objspace_t *objspace = (rb_objspace_t *)data;
VALUE obj = (VALUE)key;
VALUE table = (VALUE)val;
GC_ASSERT(RB_FL_TEST(obj, FL_FINALIZE));
GC_ASSERT(RB_BUILTIN_TYPE(val) == T_ARRAY);
- rb_gc_run_obj_finalizer(rb_gc_impl_object_id(objspace, obj), RARRAY_LEN(table), get_final, (void *)table);
+ rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), RARRAY_LEN(table) - 1, get_final, (void *)table);
FL_UNSET(obj, FL_FINALIZE);
@@ -3084,7 +3217,7 @@ rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
}
while (finalizer_table->num_entries) {
- st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, (st_data_t)objspace);
+ st_foreach(finalizer_table, rb_gc_impl_shutdown_call_finalizer_i, 0);
}
/* run finalizers */
@@ -3188,7 +3321,7 @@ gc_setup_mark_bits(struct heap_page *page)
}
static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
-static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size);
+static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct heap_page *src_page, struct heap_page *dest_page);
#if defined(_WIN32)
enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
@@ -3271,7 +3404,7 @@ try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page,
objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++;
objspace->rcompactor.total_moved++;
- gc_move(objspace, src, dest, src_page->slot_size, free_page->slot_size);
+ gc_move(objspace, src, dest, src_page, free_page);
gc_pin(objspace, src);
free_page->free_slots--;
@@ -3316,7 +3449,7 @@ read_barrier_handler(uintptr_t address)
rb_bug("read_barrier_handler: segmentation fault at %p", (void *)address);
}
- int lev = rb_gc_vm_lock();
+ int lev = RB_GC_VM_LOCK();
{
unlock_page_body(objspace, page_body);
@@ -3324,7 +3457,7 @@ read_barrier_handler(uintptr_t address)
invalidate_moved_page(objspace, GET_HEAP_PAGE(address));
}
- rb_gc_vm_unlock(lev);
+ RB_GC_VM_UNLOCK(lev);
}
#endif
@@ -3509,18 +3642,35 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit
{
struct heap_page *sweep_page = ctx->page;
short slot_size = sweep_page->slot_size;
- short slot_bits = slot_size / BASE_SLOT_SIZE;
- GC_ASSERT(slot_bits > 0);
do {
VALUE vp = (VALUE)p;
- GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
+ GC_ASSERT(vp % sizeof(VALUE) == 0);
rb_asan_unpoison_object(vp, false);
if (bitset & 1) {
switch (BUILTIN_TYPE(vp)) {
- default: /* majority case */
- gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
+ case T_MOVED:
+ if (objspace->flags.during_compacting) {
+ /* The sweep cursor shouldn't have made it to any
+ * T_MOVED slots while the compact flag is enabled.
+ * The sweep cursor and compact cursor move in
+ * opposite directions, and when they meet references will
+ * get updated and "during_compacting" should get disabled */
+ rb_bug("T_MOVED shouldn't be seen until compaction is finished");
+ }
+ gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp));
+ ctx->empty_slots++;
+ heap_page_add_freeobj(objspace, sweep_page, vp);
+ break;
+ case T_ZOMBIE:
+ /* already counted */
+ break;
+ case T_NONE:
+ ctx->empty_slots++; /* already freed */
+ break;
+
+ default:
#if RGENGC_CHECK_MODE
if (!is_full_marking(objspace)) {
if (RVALUE_OLD_P(objspace, vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
@@ -3528,8 +3678,6 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit
}
#endif
- if (RVALUE_WB_UNPROTECTED(objspace, vp)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(vp), vp);
-
#if RGENGC_CHECK_MODE
#define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp))
CHECK(RVALUE_WB_UNPROTECTED);
@@ -3539,49 +3687,31 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit
#undef CHECK
#endif
- rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ);
-
- bool has_object_id = FL_TEST(vp, FL_SEEN_OBJ_ID);
- rb_gc_obj_free_vm_weak_references(vp);
- if (rb_gc_obj_free(objspace, vp)) {
- if (has_object_id) {
- obj_free_object_id(objspace, vp);
- }
- // always add free slots back to the swept pages freelist,
- // so that if we're compacting, we can re-use the slots
- (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE);
+ if (!rb_gc_obj_needs_cleanup_p(vp)) {
+ (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
heap_page_add_freeobj(objspace, sweep_page, vp);
- gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp));
+ gc_report(3, objspace, "page_sweep: %s (fast path) added to freelist\n", rb_obj_info(vp));
ctx->freed_slots++;
}
else {
- ctx->final_slots++;
- }
- break;
+ gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
- case T_MOVED:
- if (objspace->flags.during_compacting) {
- /* The sweep cursor shouldn't have made it to any
- * T_MOVED slots while the compact flag is enabled.
- * The sweep cursor and compact cursor move in
- * opposite directions, and when they meet references will
- * get updated and "during_compacting" should get disabled */
- rb_bug("T_MOVED shouldn't be seen until compaction is finished");
+ rb_gc_obj_free_vm_weak_references(vp);
+ if (rb_gc_obj_free(objspace, vp)) {
+ (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
+ heap_page_add_freeobj(objspace, sweep_page, vp);
+ gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp));
+ ctx->freed_slots++;
+ }
+ else {
+ ctx->final_slots++;
+ }
}
- gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp));
- ctx->empty_slots++;
- heap_page_add_freeobj(objspace, sweep_page, vp);
- break;
- case T_ZOMBIE:
- /* already counted */
- break;
- case T_NONE:
- ctx->empty_slots++; /* already freed */
break;
}
}
p += slot_size;
- bitset >>= slot_bits;
+ bitset >>= 1;
} while (bitset);
}
@@ -3606,34 +3736,33 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context
p = (uintptr_t)sweep_page->start;
bits = sweep_page->mark_bits;
+ short slot_size = sweep_page->slot_size;
+ int total_slots = sweep_page->total_slots;
+ int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
- int page_rvalue_count = sweep_page->total_slots * (sweep_page->slot_size / BASE_SLOT_SIZE);
- int out_of_range_bits = (NUM_IN_PAGE(p) + page_rvalue_count) % BITS_BITLENGTH;
- if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64
- bits[BITMAP_INDEX(p) + page_rvalue_count / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
+ int out_of_range_bits = total_slots % BITS_BITLENGTH;
+ if (out_of_range_bits != 0) {
+ bits[bitmap_plane_count - 1] |= ~(((bits_t)1 << out_of_range_bits) - 1);
}
- /* The last bitmap plane may not be used if the last plane does not
- * have enough space for the slot_size. In that case, the last plane must
- * be skipped since none of the bits will be set. */
- int bitmap_plane_count = CEILDIV(NUM_IN_PAGE(p) + page_rvalue_count, BITS_BITLENGTH);
- GC_ASSERT(bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT - 1 ||
- bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT);
-
- // Skip out of range slots at the head of the page
- bitset = ~bits[0];
- bitset >>= NUM_IN_PAGE(p);
- if (bitset) {
- gc_sweep_plane(objspace, heap, p, bitset, ctx);
+ // Clear wb_unprotected and age bits for all unmarked slots
+ {
+ bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits;
+ bits_t *age_bits = sweep_page->age_bits;
+ for (int i = 0; i < bitmap_plane_count; i++) {
+ bits_t unmarked = ~bits[i];
+ wb_unprotected_bits[i] &= ~unmarked;
+ age_bits[i * 2] &= ~unmarked;
+ age_bits[i * 2 + 1] &= ~unmarked;
+ }
}
- p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
- for (int i = 1; i < bitmap_plane_count; i++) {
+ for (int i = 0; i < bitmap_plane_count; i++) {
bitset = ~bits[i];
if (bitset) {
gc_sweep_plane(objspace, heap, p, bitset, ctx);
}
- p += BITS_BITLENGTH * BASE_SLOT_SIZE;
+ p += BITS_BITLENGTH * slot_size;
}
if (!heap->compact_cursor) {
@@ -3735,6 +3864,9 @@ static void
gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
{
heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node);
+ if (heap->sweeping_page) {
+ objspace->sweeping_heap_count++;
+ }
heap->free_pages = NULL;
heap->pooled_pages = NULL;
if (!objspace->flags.immediate_sweep) {
@@ -3758,13 +3890,19 @@ static int compare_pinned_slots(const void *left, const void *right, void *d);
static void
gc_ractor_newobj_cache_clear(void *c, void *data)
{
+ rb_objspace_t *objspace = rb_gc_get_objspace();
rb_ractor_newobj_cache_t *newobj_cache = c;
newobj_cache->incremental_mark_step_allocated_slots = 0;
for (size_t heap_idx = 0; heap_idx < HEAP_COUNT; heap_idx++) {
+
rb_ractor_newobj_heap_cache_t *cache = &newobj_cache->heap_caches[heap_idx];
+ rb_heap_t *heap = &heaps[heap_idx];
+ RUBY_ATOMIC_SIZE_ADD(heap->total_allocated_objects, cache->allocated_objects_count);
+ cache->allocated_objects_count = 0;
+
struct heap_page *page = cache->using_page;
struct free_slot *freelist = cache->freelist;
RUBY_DEBUG_LOG("ractor using_page:%p freelist:%p", (void *)page, (void *)freelist);
@@ -3777,11 +3915,70 @@ gc_ractor_newobj_cache_clear(void *c, void *data)
}
static void
+gc_sweep_freeobj_hooks_page(rb_objspace_t *objspace, struct heap_page *page)
+{
+ bits_t *bits = page->mark_bits;
+ uintptr_t p = (uintptr_t)page->start;
+ short slot_size = page->slot_size;
+ int total_slots = page->total_slots;
+ int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
+
+ int out_of_range_bits = total_slots % BITS_BITLENGTH;
+ bits_t last_plane_mask = (out_of_range_bits != 0)
+ ? ~(((bits_t)1 << out_of_range_bits) - 1)
+ : 0;
+
+ for (int j = 0; j < bitmap_plane_count; j++) {
+ bits_t bitset = ~bits[j];
+ if (j == bitmap_plane_count - 1) {
+ bitset &= ~last_plane_mask;
+ }
+
+ uintptr_t pp = p;
+ while (bitset) {
+ if (bitset & 1) {
+ VALUE vp = (VALUE)pp;
+ asan_unpoisoning_object(vp) {
+ switch (BUILTIN_TYPE(vp)) {
+ case T_NONE:
+ case T_ZOMBIE:
+ case T_MOVED:
+ break;
+ default:
+ rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ);
+ break;
+ }
+ }
+ }
+ pp += slot_size;
+ bitset >>= 1;
+ }
+ p += BITS_BITLENGTH * slot_size;
+ }
+}
+
+static void
+gc_sweep_freeobj_hooks(rb_objspace_t *objspace)
+{
+ for (int i = 0; i < HEAP_COUNT; i++) {
+ rb_heap_t *heap = &heaps[i];
+ struct heap_page *page = NULL;
+
+ ccan_list_for_each(&heap->pages, page, page_node) {
+ gc_sweep_freeobj_hooks_page(objspace, page);
+ }
+ }
+}
+
+static void
gc_sweep_start(rb_objspace_t *objspace)
{
gc_mode_transition(objspace, gc_mode_sweeping);
objspace->rincgc.pooled_slots = 0;
- objspace->heap_pages.allocatable_slots = 0;
+
+ if (RB_UNLIKELY(objspace->hook_events & RUBY_INTERNAL_EVENT_FREEOBJ)) {
+ gc_sweep_freeobj_hooks(objspace);
+ }
#if GC_CAN_COMPILE_COMPACTION
if (objspace->flags.during_compacting) {
@@ -3813,12 +4010,12 @@ gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap)
size_t total_slots = heap->total_slots;
size_t swept_slots = heap->freed_slots + heap->empty_slots;
- size_t init_slots = gc_params.heap_init_slots[heap - heaps];
+ size_t init_slots = gc_params.heap_init_bytes / heap->slot_size;
size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio);
if (swept_slots < min_free_slots &&
/* The heap is a growth heap if it freed more slots than had empty slots. */
- (heap->empty_slots == 0 || heap->freed_slots > heap->empty_slots)) {
+ ((heap->empty_slots == 0 && total_slots > 0) || heap->freed_slots > heap->empty_slots)) {
/* If we don't have enough slots and we have pages on the tomb heap, move
* pages from the tomb heap to the eden heap. This may prevent page
* creation thrashing (frequently allocating and deallocting pages) and
@@ -3834,12 +4031,15 @@ gc_sweep_finish_heap(rb_objspace_t *objspace, rb_heap_t *heap)
if (swept_slots < min_free_slots) {
/* Grow this heap if we are in a major GC or if we haven't run at least
- * RVALUE_OLD_AGE minor GC since the last major GC. */
+ * RVALUE_OLD_AGE minor GC since the last major GC. */
if (is_full_marking(objspace) ||
objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
- heap_allocatable_slots_expand(objspace, heap, swept_slots, heap->total_slots);
+ if (objspace->heap_pages.allocatable_bytes < min_free_slots * heap->slot_size) {
+ heap_allocatable_bytes_expand(objspace, heap, swept_slots, heap->total_slots, heap->slot_size);
+ }
}
- else {
+ else if (swept_slots < min_free_slots * 7 / 8 &&
+ objspace->heap_pages.allocatable_bytes < (min_free_slots * 7 / 8 - swept_slots) * heap->slot_size) {
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
heap->force_major_gc_count++;
}
@@ -3875,6 +4075,13 @@ gc_sweep_finish(rb_objspace_t *objspace)
}
}
+ (void)gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.counters);
+#if RGENGC_ESTIMATE_OLDMALLOC
+ if (objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) {
+ (void)gc_malloc_counters_snapshot(objspace, &objspace->malloc_counters.oldcounters);
+ }
+#endif
+
rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_SWEEP);
gc_mode_transition(objspace, gc_mode_none);
@@ -3887,9 +4094,10 @@ static int
gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *sweep_page = heap->sweeping_page;
- int unlink_limit = GC_SWEEP_PAGES_FREEABLE_PER_STEP;
int swept_slots = 0;
int pooled_slots = 0;
+ int sweep_budget = GC_INCREMENTAL_SWEEP_BYTES / heap->slot_size;
+ int pool_budget = GC_INCREMENTAL_SWEEP_POOL_BYTES / heap->slot_size;
if (sweep_page == NULL) return FALSE;
@@ -3911,11 +4119,7 @@ gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
- if (free_slots == sweep_page->total_slots &&
- heap_pages_freeable_pages > 0 &&
- unlink_limit > 0) {
- heap_pages_freeable_pages--;
- unlink_limit--;
+ if (free_slots == sweep_page->total_slots) {
/* There are no living objects, so move this page to the global empty pages. */
heap_unlink_page(objspace, heap, sweep_page);
@@ -3939,14 +4143,14 @@ gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
heap->freed_slots += ctx.freed_slots;
heap->empty_slots += ctx.empty_slots;
- if (pooled_slots < GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT) {
+ if (pooled_slots < pool_budget) {
heap_add_poolpage(objspace, heap, sweep_page);
pooled_slots += free_slots;
}
else {
heap_add_freepage(heap, sweep_page);
swept_slots += free_slots;
- if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) {
+ if (swept_slots > sweep_budget) {
break;
}
}
@@ -3957,6 +4161,8 @@ gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
} while ((sweep_page = heap->sweeping_page));
if (!heap->sweeping_page) {
+ objspace->sweeping_heap_count--;
+ GC_ASSERT(objspace->sweeping_heap_count >= 0);
gc_sweep_finish_heap(objspace, heap);
if (!has_sweeping_pages(objspace)) {
@@ -3993,12 +4199,29 @@ gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *sweep_heap)
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
- if (!gc_sweep_step(objspace, heap)) {
- /* sweep_heap requires a free slot but sweeping did not yield any
- * and we cannot allocate a new page. */
- if (heap == sweep_heap && objspace->heap_pages.allocatable_slots == 0) {
+ if (gc_sweep_step(objspace, heap)) {
+ GC_ASSERT(heap->free_pages != NULL);
+ }
+ else if (heap == sweep_heap) {
+ if (objspace->empty_pages_count > 0 || objspace->heap_pages.allocatable_bytes > 0) {
+ /* [Bug #21548]
+ *
+ * If this heap is the heap we want to sweep, but we weren't able
+ * to free any slots, but we also either have empty pages or could
+ * allocate new pages, then we want to preemptively claim a page
+ * because it's possible that sweeping another heap will call
+ * gc_sweep_finish_heap, which may use up all of the
+ * empty/allocatable pages. If other heaps are not finished sweeping
+ * then we do not finish this GC and we will end up triggering a new
+ * GC cycle during this GC phase. */
+ heap_page_allocate_and_initialize(objspace, heap);
+
+ GC_ASSERT(heap->free_pages != NULL);
+ }
+ else {
/* Not allowed to create a new page so finish sweeping. */
gc_sweep_rest(objspace);
+ GC_ASSERT(gc_mode(objspace) == gc_mode_none);
break;
}
}
@@ -4042,22 +4265,13 @@ invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_
CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
object = rb_gc_impl_location(objspace, forwarding_object);
-
- uint32_t original_shape_id = 0;
- if (RB_TYPE_P(object, T_OBJECT)) {
- original_shape_id = RMOVED(forwarding_object)->original_shape_id;
- }
-
- gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object)->slot_size, page->slot_size);
+ gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object), page);
/* forwarding_object is now our actual object, and "object"
* is the free slot for the original page */
- if (original_shape_id) {
- rb_gc_set_shape(forwarding_object, original_shape_id);
- }
-
struct heap_page *orig_page = GET_HEAP_PAGE(object);
orig_page->free_slots++;
+ RVALUE_AGE_SET_BITMAP(object, 0);
heap_page_add_freeobj(objspace, orig_page, object);
GC_ASSERT(RVALUE_MARKED(objspace, forwarding_object));
@@ -4065,7 +4279,7 @@ invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_
GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
}
}
- p += BASE_SLOT_SIZE;
+ p += page->slot_size;
bitset >>= 1;
} while (bitset);
}
@@ -4077,25 +4291,21 @@ invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
int i;
bits_t *mark_bits, *pin_bits;
bits_t bitset;
+ short slot_size = page->slot_size;
+ int total_slots = page->total_slots;
+ int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
uintptr_t p = page->start;
- // Skip out of range slots at the head of the page
- bitset = pin_bits[0] & ~mark_bits[0];
- bitset >>= NUM_IN_PAGE(p);
- invalidate_moved_plane(objspace, page, p, bitset);
- p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
-
- for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
+ for (i=0; i < bitmap_plane_count; i++) {
/* Moved objects are pinned but never marked. We reuse the pin bits
* to indicate there is a moved object in this slot. */
bitset = pin_bits[i] & ~mark_bits[i];
-
invalidate_moved_plane(objspace, page, p, bitset);
- p += BITS_BITLENGTH * BASE_SLOT_SIZE;
+ p += BITS_BITLENGTH * slot_size;
}
}
#endif
@@ -4367,15 +4577,11 @@ init_mark_stack(mark_stack_t *stack)
static void
rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
{
- const VALUE old_parent = objspace->rgengc.parent_object;
-
- if (old_parent) { /* parent object is old */
+ if (objspace->rgengc.parent_object_old_p) {
if (RVALUE_WB_UNPROTECTED(objspace, obj) || !RVALUE_OLD_P(objspace, obj)) {
- rgengc_remember(objspace, old_parent);
+ rgengc_remember(objspace, objspace->rgengc.parent_object);
}
}
-
- GC_ASSERT(old_parent == objspace->rgengc.parent_object);
}
static inline int
@@ -4406,8 +4612,16 @@ gc_aging(rb_objspace_t *objspace, VALUE obj)
if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
if (!RVALUE_OLD_P(objspace, obj)) {
- gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj));
- RVALUE_AGE_INC(objspace, obj);
+ int t = BUILTIN_TYPE(obj);
+ if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) {
+ gc_report(3, objspace, "gc_aging: YOUNG class: %s\n", rb_obj_info(obj));
+ RVALUE_AGE_SET(obj, RVALUE_OLD_AGE);
+ RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
+ }
+ else {
+ gc_report(3, objspace, "gc_aging: YOUNG: %s\n", rb_obj_info(obj));
+ RVALUE_AGE_INC(objspace, obj);
+ }
}
else if (is_full_marking(objspace)) {
GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
@@ -4431,32 +4645,44 @@ gc_grey(rb_objspace_t *objspace, VALUE obj)
MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
}
+ if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) {
+ rb_darray_append_without_gc(&objspace->weak_references, obj);
+ }
+
push_mark_stack(&objspace->mark_stack, obj);
}
+static inline void
+gc_mark_check_t_none(rb_objspace_t *objspace, VALUE obj)
+{
+ if (RB_UNLIKELY(BUILTIN_TYPE(obj) == T_NONE)) {
+ enum {info_size = 256};
+ char obj_info_buf[info_size];
+ rb_raw_obj_info(obj_info_buf, info_size, obj);
+
+ char parent_obj_info_buf[info_size];
+ rb_raw_obj_info(parent_obj_info_buf, info_size, objspace->rgengc.parent_object);
+
+ rb_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
+ }
+}
+
static void
gc_mark(rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(during_gc);
+ GC_ASSERT(!objspace->flags.during_reference_updating);
rgengc_check_relation(objspace, obj);
if (!gc_mark_set(objspace, obj)) return; /* already marked */
if (0) { // for debug GC marking miss
- if (objspace->rgengc.parent_object) {
- RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
- (void *)obj, obj_type_name(obj),
- (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
- }
- else {
- RUBY_DEBUG_LOG("%p (%s)", (void *)obj, obj_type_name(obj));
- }
+ RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
+ (void *)obj, obj_type_name(obj),
+ (void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
}
- if (RB_UNLIKELY(RB_TYPE_P(obj, T_NONE))) {
- rb_obj_info_dump(obj);
- rb_bug("try to mark T_NONE object"); /* check here will help debugging */
- }
+ gc_mark_check_t_none(objspace, obj);
gc_aging(objspace, obj);
gc_grey(objspace, obj);
@@ -4493,7 +4719,10 @@ rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
GC_ASSERT(objspace->flags.during_compacting);
GC_ASSERT(during_gc);
- *ptr = rb_gc_impl_location(objspace, *ptr);
+ VALUE destination = rb_gc_impl_location(objspace, *ptr);
+ if (destination != *ptr) {
+ *ptr = destination;
+ }
}
else {
gc_mark(objspace, *ptr);
@@ -4527,10 +4756,10 @@ rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
asan_unpoisoning_object(obj) {
/* Garbage can live on the stack, so do not mark or pin */
switch (BUILTIN_TYPE(obj)) {
- case T_ZOMBIE:
- case T_NONE:
+ case T_ZOMBIE:
+ case T_NONE:
break;
- default:
+ default:
gc_mark_and_pin(objspace, obj);
break;
}
@@ -4538,64 +4767,34 @@ rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
}
}
-void
-rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr)
+static int
+pin_value(st_data_t key, st_data_t value, st_data_t data)
{
- rb_objspace_t *objspace = objspace_ptr;
-
- GC_ASSERT(objspace->rgengc.parent_object == 0 || FL_TEST(objspace->rgengc.parent_object, FL_WB_PROTECTED));
-
- VALUE obj = *ptr;
-
- if (RB_UNLIKELY(RB_TYPE_P(obj, T_NONE))) {
- rb_obj_info_dump(obj);
- rb_bug("try to mark T_NONE object");
- }
-
- /* If we are in a minor GC and the other object is old, then obj should
- * already be marked and cannot be reclaimed in this GC cycle so we don't
- * need to add it to the weak references list. */
- if (!is_full_marking(objspace) && RVALUE_OLD_P(objspace, obj)) {
- GC_ASSERT(RVALUE_MARKED(objspace, obj));
- GC_ASSERT(!objspace->flags.during_compacting);
-
- return;
- }
-
- rgengc_check_relation(objspace, obj);
-
- rb_darray_append_without_gc(&objspace->weak_references, ptr);
+ rb_gc_impl_mark_and_pin((void *)data, (VALUE)value);
- objspace->profile.weak_references_count++;
+ return ST_CONTINUE;
}
-void
-rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr)
+static inline void
+gc_mark_set_parent_raw(rb_objspace_t *objspace, VALUE obj, bool old_p)
{
- rb_objspace_t *objspace = objspace_ptr;
-
- /* If we're not incremental marking, then the state of the objects can't
- * change so we don't need to do anything. */
- if (!is_incremental_marking(objspace)) return;
- /* If parent_obj has not been marked, then ptr has not yet been marked
- * weak, so we don't need to do anything. */
- if (!RVALUE_MARKED(objspace, parent_obj)) return;
-
- VALUE **ptr_ptr;
- rb_darray_foreach(objspace->weak_references, i, ptr_ptr) {
- if (*ptr_ptr == ptr) {
- *ptr_ptr = NULL;
- break;
- }
- }
+ asan_unpoison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object), false);
+ asan_unpoison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p), false);
+ objspace->rgengc.parent_object = obj;
+ objspace->rgengc.parent_object_old_p = old_p;
}
-static int
-pin_value(st_data_t key, st_data_t value, st_data_t data)
+static inline void
+gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
{
- rb_gc_impl_mark_and_pin((void *)data, (VALUE)value);
+ gc_mark_set_parent_raw(objspace, obj, RVALUE_OLD_P(objspace, obj));
+}
- return ST_CONTINUE;
+static inline void
+gc_mark_set_parent_invalid(rb_objspace_t *objspace)
+{
+ asan_poison_memory_region(&objspace->rgengc.parent_object, sizeof(objspace->rgengc.parent_object));
+ asan_poison_memory_region(&objspace->rgengc.parent_object_old_p, sizeof(objspace->rgengc.parent_object_old_p));
}
static void
@@ -4606,29 +4805,17 @@ mark_roots(rb_objspace_t *objspace, const char **categoryp)
} while (0)
MARK_CHECKPOINT("objspace");
- objspace->rgengc.parent_object = Qfalse;
+ gc_mark_set_parent_raw(objspace, Qundef, false);
if (finalizer_table != NULL) {
st_foreach(finalizer_table, pin_value, (st_data_t)objspace);
}
- st_foreach(objspace->obj_to_id_tbl, gc_mark_tbl_no_pin_i, (st_data_t)objspace);
-
if (stress_to_class) rb_gc_mark(stress_to_class);
rb_gc_save_machine_context();
rb_gc_mark_roots(objspace, categoryp);
-}
-
-static inline void
-gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
-{
- if (RVALUE_OLD_P(objspace, obj)) {
- objspace->rgengc.parent_object = obj;
- }
- else {
- objspace->rgengc.parent_object = Qfalse;
- }
+ gc_mark_set_parent_invalid(objspace);
}
static void
@@ -4636,6 +4823,7 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
{
gc_mark_set_parent(objspace, obj);
rb_gc_mark_children(objspace, obj);
+ gc_mark_set_parent_invalid(objspace);
}
/**
@@ -4921,10 +5109,22 @@ gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
static void
gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
{
- size_t saved_malloc_increase = objspace->malloc_params.increase;
+ MALLOC_COUNTERS_LOCK(objspace);
+ struct gc_malloc_bytes saved_malloc = {
+ .malloc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc),
+ .free = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free),
+ .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc_at_last_gc),
+ .free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.counters.free_at_last_gc),
+ };
#if RGENGC_ESTIMATE_OLDMALLOC
- size_t saved_oldmalloc_increase = objspace->rgengc.oldmalloc_increase;
+ struct gc_malloc_bytes saved_oldmalloc = {
+ .malloc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.malloc),
+ .free = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.free),
+ .malloc_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.malloc_at_last_gc),
+ .free_at_last_gc = gc_counter_load_relaxed(&objspace->malloc_counters.oldcounters.free_at_last_gc),
+ };
#endif
+ MALLOC_COUNTERS_UNLOCK(objspace);
VALUE already_disabled = rb_objspace_gc_disable(objspace);
objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
@@ -4944,10 +5144,18 @@ gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func,
objspace->rgengc.allrefs_table = 0;
if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
- objspace->malloc_params.increase = saved_malloc_increase;
+ MALLOC_COUNTERS_LOCK(objspace);
+ gc_counter_store_release(&objspace->malloc_counters.counters.malloc, saved_malloc.malloc);
+ gc_counter_store_release(&objspace->malloc_counters.counters.free, saved_malloc.free);
+ gc_counter_store_release(&objspace->malloc_counters.counters.malloc_at_last_gc, saved_malloc.malloc_at_last_gc);
+ gc_counter_store_release(&objspace->malloc_counters.counters.free_at_last_gc, saved_malloc.free_at_last_gc);
#if RGENGC_ESTIMATE_OLDMALLOC
- objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase;
+ gc_counter_store_release(&objspace->malloc_counters.oldcounters.malloc, saved_oldmalloc.malloc);
+ gc_counter_store_release(&objspace->malloc_counters.oldcounters.free, saved_oldmalloc.free);
+ gc_counter_store_release(&objspace->malloc_counters.oldcounters.malloc_at_last_gc, saved_oldmalloc.malloc_at_last_gc);
+ gc_counter_store_release(&objspace->malloc_counters.oldcounters.free_at_last_gc, saved_oldmalloc.free_at_last_gc);
#endif
+ MALLOC_COUNTERS_UNLOCK(objspace);
}
#endif /* RGENGC_CHECK_MODE >= 4 */
@@ -5036,6 +5244,10 @@ verify_internal_consistency_i(void *page_start, void *page_end, size_t stride,
rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data);
}
+ if (!is_marking(objspace) && rb_gc_obj_shareable_p(obj)) {
+ rb_gc_verify_shareable(obj);
+ }
+
if (is_incremental_marking(objspace)) {
if (RVALUE_BLACK_P(objspace, obj)) {
/* reachable objects from black objects should be black or grey objects */
@@ -5201,6 +5413,8 @@ gc_verify_internal_consistency_(rb_objspace_t *objspace)
/* check counters */
+ ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache());
+
if (!is_lazy_sweeping(objspace) &&
!finalizing &&
!rb_gc_multi_ractor_p()) {
@@ -5255,7 +5469,7 @@ gc_verify_internal_consistency(void *objspace_ptr)
{
rb_objspace_t *objspace = objspace_ptr;
- unsigned int lev = rb_gc_vm_lock();
+ unsigned int lev = RB_GC_VM_LOCK();
{
rb_gc_vm_barrier(); // stop other ractors
@@ -5266,7 +5480,7 @@ gc_verify_internal_consistency(void *objspace_ptr)
}
during_gc = prev_during_gc;
}
- rb_gc_vm_unlock(lev);
+ RB_GC_VM_UNLOCK(lev);
}
static void
@@ -5313,7 +5527,7 @@ gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
}
static inline void
-gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits)
+gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits, short slot_size)
{
if (bits) {
do {
@@ -5323,7 +5537,7 @@ gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits
GC_ASSERT(RVALUE_MARKED(objspace, (VALUE)p));
gc_mark_children(objspace, (VALUE)p);
}
- p += BASE_SLOT_SIZE;
+ p += slot_size;
bits >>= 1;
} while (bits);
}
@@ -5338,48 +5552,63 @@ gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap)
bits_t *mark_bits = page->mark_bits;
bits_t *wbun_bits = page->wb_unprotected_bits;
uintptr_t p = page->start;
+ short slot_size = page->slot_size;
+ int total_slots = page->total_slots;
+ int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
size_t j;
- bits_t bits = mark_bits[0] & wbun_bits[0];
- bits >>= NUM_IN_PAGE(p);
- gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
- p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
-
- for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
+ for (j=0; j<(size_t)bitmap_plane_count; j++) {
bits_t bits = mark_bits[j] & wbun_bits[j];
-
- gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
- p += BITS_BITLENGTH * BASE_SLOT_SIZE;
+ gc_marks_wb_unprotected_objects_plane(objspace, p, bits, slot_size);
+ p += BITS_BITLENGTH * slot_size;
}
}
gc_mark_stacked_objects_all(objspace);
}
-static void
-gc_update_weak_references(rb_objspace_t *objspace)
+void
+rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
{
- size_t retained_weak_references_count = 0;
- VALUE **ptr_ptr;
- rb_darray_foreach(objspace->weak_references, i, ptr_ptr) {
- if (!*ptr_ptr) continue;
+ FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE);
+}
- VALUE obj = **ptr_ptr;
+bool
+rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
+{
+ rb_objspace_t *objspace = objspace_ptr;
- if (RB_SPECIAL_CONST_P(obj)) continue;
+ bool marked = RVALUE_MARKED(objspace, obj);
- if (!RVALUE_MARKED(objspace, obj)) {
- **ptr_ptr = Qundef;
- }
- else {
- retained_weak_references_count++;
- }
+ if (marked) {
+ rgengc_check_relation(objspace, obj);
}
- objspace->profile.retained_weak_references_count = retained_weak_references_count;
+ return marked;
+}
+
+static void
+gc_update_weak_references(rb_objspace_t *objspace)
+{
+ VALUE *obj_ptr;
+ rb_darray_foreach(objspace->weak_references, i, obj_ptr) {
+ gc_mark_set_parent(objspace, *obj_ptr);
+ rb_gc_handle_weak_references(*obj_ptr);
+ gc_mark_set_parent_invalid(objspace);
+ }
+
+ size_t capa = rb_darray_capa(objspace->weak_references);
+ size_t size = rb_darray_size(objspace->weak_references);
+
+ objspace->profile.weak_references_count = size;
rb_darray_clear(objspace->weak_references);
- rb_darray_resize_capa_without_gc(&objspace->weak_references, retained_weak_references_count);
+
+ /* If the darray has capacity for more than four times the amount used, we
+ * shrink it down to half of that capacity. */
+ if (capa > size * 4) {
+ rb_darray_resize_capa_without_gc(&objspace->weak_references, size * 2);
+ }
}
static void
@@ -5438,22 +5667,27 @@ gc_marks_finish(rb_objspace_t *objspace)
/* Setup freeable slots. */
size_t total_init_slots = 0;
for (int i = 0; i < HEAP_COUNT; i++) {
- total_init_slots += gc_params.heap_init_slots[i] * r_mul;
+ total_init_slots += (gc_params.heap_init_bytes / heaps[i].slot_size) * r_mul;
}
if (max_free_slots < total_init_slots) {
max_free_slots = total_init_slots;
}
+ /* Approximate freeable pages using the average slots-per-pages across all heaps */
if (sweep_slots > max_free_slots) {
- heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
+ size_t excess_slots = sweep_slots - max_free_slots;
+ size_t total_heap_pages = heap_eden_total_pages(objspace);
+ heap_pages_freeable_pages = total_heap_pages > 0
+ ? excess_slots * total_heap_pages / total_slots
+ : 0;
}
else {
heap_pages_freeable_pages = 0;
}
- if (objspace->heap_pages.allocatable_slots == 0 && sweep_slots < min_free_slots) {
- if (!full_marking) {
+ if (objspace->heap_pages.allocatable_bytes == 0 && sweep_slots < min_free_slots) {
+ if (!full_marking && sweep_slots < min_free_slots * 7 / 8) {
if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
full_marking = TRUE;
}
@@ -5462,6 +5696,10 @@ gc_marks_finish(rb_objspace_t *objspace)
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE;
}
}
+
+ if (full_marking) {
+ heap_allocatable_bytes_expand(objspace, NULL, sweep_slots, total_slots, heaps[0].slot_size);
+ }
}
if (full_marking) {
@@ -5483,8 +5721,8 @@ gc_marks_finish(rb_objspace_t *objspace)
gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, "
"old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, "
- "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" slots, next GC: %s)\n",
- objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_slots,
+ "sweep %"PRIdSIZE" slots, allocatable %"PRIdSIZE" bytes, next GC: %s)\n",
+ objspace->marked_slots, objspace->rgengc.old_objects, objspace_available_slots(objspace), sweep_slots, objspace->heap_pages.allocatable_bytes,
gc_needs_major_flags ? "major" : "minor");
}
@@ -5509,10 +5747,9 @@ gc_compact_destination_pool(rb_objspace_t *objspace, rb_heap_t *src_pool, VALUE
return src_pool;
}
- size_t idx = 0;
- if (rb_gc_impl_size_allocatable_p(obj_size)) {
- idx = heap_idx_for_size(obj_size);
- }
+ GC_ASSERT(rb_gc_impl_size_allocatable_p(obj_size));
+
+ size_t idx = heap_idx_for_size(obj_size);
return &heaps[idx];
}
@@ -5524,25 +5761,10 @@ gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src)
GC_ASSERT(gc_is_moveable_obj(objspace, src));
rb_heap_t *dest_pool = gc_compact_destination_pool(objspace, heap, src);
- uint32_t orig_shape = 0;
- uint32_t new_shape = 0;
-
if (gc_compact_heap_cursors_met_p(dest_pool)) {
return dest_pool != heap;
}
- if (RB_TYPE_P(src, T_OBJECT)) {
- orig_shape = rb_gc_get_shape(src);
-
- if (dest_pool != heap) {
- new_shape = rb_gc_rebuild_shape(src, dest_pool - heaps);
-
- if (new_shape == 0) {
- dest_pool = heap;
- }
- }
- }
-
while (!try_move(objspace, dest_pool, dest_pool->free_pages, src)) {
struct gc_sweep_context ctx = {
.page = dest_pool->sweeping_page,
@@ -5568,14 +5790,6 @@ gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, VALUE src)
}
}
- if (orig_shape != 0) {
- if (new_shape != 0) {
- VALUE dest = rb_gc_impl_location(objspace, src);
- rb_gc_set_shape(dest, new_shape);
- }
- RMOVED(src)->original_shape_id = orig_shape;
- }
-
return true;
}
@@ -5583,12 +5797,10 @@ static bool
gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page)
{
short slot_size = page->slot_size;
- short slot_bits = slot_size / BASE_SLOT_SIZE;
- GC_ASSERT(slot_bits > 0);
do {
VALUE vp = (VALUE)p;
- GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
+ GC_ASSERT(vp % sizeof(VALUE) == 0);
if (bitset & 1) {
objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
@@ -5601,7 +5813,7 @@ gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t b
}
}
p += slot_size;
- bitset >>= slot_bits;
+ bitset >>= 1;
} while (bitset);
return true;
@@ -5616,26 +5828,21 @@ gc_compact_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page
bits_t *mark_bits, *pin_bits;
bits_t bitset;
uintptr_t p = page->start;
+ short slot_size = page->slot_size;
+ int total_slots = page->total_slots;
+ int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
- // objects that can be moved are marked and not pinned
- bitset = (mark_bits[0] & ~pin_bits[0]);
- bitset >>= NUM_IN_PAGE(p);
- if (bitset) {
- if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page))
- return false;
- }
- p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
-
- for (int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
+ for (int j = 0; j < bitmap_plane_count; j++) {
+ // objects that can be moved are marked and not pinned
bitset = (mark_bits[j] & ~pin_bits[j]);
if (bitset) {
if (!gc_compact_plane(objspace, heap, (uintptr_t)p, bitset, page))
return false;
}
- p += BITS_BITLENGTH * BASE_SLOT_SIZE;
+ p += BITS_BITLENGTH * slot_size;
}
return true;
@@ -5812,7 +6019,6 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark)
static bool
gc_marks(rb_objspace_t *objspace, int full_mark)
{
- gc_prof_mark_timer_start(objspace);
gc_marking_enter(objspace);
bool marking_finished = false;
@@ -5833,7 +6039,6 @@ gc_marks(rb_objspace_t *objspace, int full_mark)
#endif
gc_marking_exit(objspace);
- gc_prof_mark_timer_stop(objspace);
return marking_finished;
}
@@ -5922,7 +6127,7 @@ rgengc_remember(rb_objspace_t *objspace, VALUE obj)
#endif
static inline void
-rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset)
+rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset, short slot_size)
{
if (bitset) {
do {
@@ -5933,8 +6138,12 @@ rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitse
GC_ASSERT(RVALUE_OLD_P(objspace, obj) || RVALUE_WB_UNPROTECTED(objspace, obj));
gc_mark_children(objspace, obj);
+
+ if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) {
+ rb_darray_append_without_gc(&objspace->weak_references, obj);
+ }
}
- p += BASE_SLOT_SIZE;
+ p += slot_size;
bitset >>= 1;
} while (bitset);
}
@@ -5953,6 +6162,9 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
ccan_list_for_each(&heap->pages, page, page_node) {
if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) {
uintptr_t p = page->start;
+ short slot_size = page->slot_size;
+ int total_slots = page->total_slots;
+ int bitmap_plane_count = CEILDIV(total_slots, BITS_BITLENGTH);
bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t *remembered_bits = page->remembered_bits;
bits_t *uncollectible_bits = page->uncollectible_bits;
@@ -5962,21 +6174,16 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
else if (page->flags.has_remembered_objects) has_old++;
else if (page->flags.has_uncollectible_wb_unprotected_objects) has_shady++;
#endif
- for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
+ for (j=0; j < (size_t)bitmap_plane_count; j++) {
bits[j] = remembered_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]);
remembered_bits[j] = 0;
}
page->flags.has_remembered_objects = FALSE;
- bitset = bits[0];
- bitset >>= NUM_IN_PAGE(p);
- rgengc_rememberset_mark_plane(objspace, p, bitset);
- p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
-
- for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
+ for (j=0; j < (size_t)bitmap_plane_count; j++) {
bitset = bits[j];
- rgengc_rememberset_mark_plane(objspace, p, bitset);
- p += BITS_BITLENGTH * BASE_SLOT_SIZE;
+ rgengc_rememberset_mark_plane(objspace, p, bitset, slot_size);
+ p += BITS_BITLENGTH * slot_size;
}
}
#if PROFILE_REMEMBERSET_MARK
@@ -6023,11 +6230,11 @@ gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
/* mark `a' and remember (default behavior) */
if (!RVALUE_REMEMBERED(objspace, a)) {
- int lev = rb_gc_vm_lock_no_barrier();
+ int lev = RB_GC_VM_LOCK_NO_BARRIER();
{
rgengc_remember(objspace, a);
}
- rb_gc_vm_unlock_no_barrier(lev);
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", rb_obj_info(a), rb_obj_info(b));
}
@@ -6041,9 +6248,11 @@ gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
{
gc_mark_set_parent(objspace, parent);
rgengc_check_relation(objspace, obj);
- if (gc_mark_set(objspace, obj) == FALSE) return;
- gc_aging(objspace, obj);
- gc_grey(objspace, obj);
+ if (gc_mark_set(objspace, obj) != FALSE) {
+ gc_aging(objspace, obj);
+ gc_grey(objspace, obj);
+ }
+ gc_mark_set_parent_invalid(objspace);
}
NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
@@ -6075,11 +6284,15 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
{
rb_objspace_t *objspace = objspace_ptr;
- if (RGENGC_CHECK_MODE) {
- if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
- if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
- }
+#if RGENGC_CHECK_MODE
+ if (SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const: %"PRIxVALUE, a);
+ if (SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const: %"PRIxVALUE, b);
+#else
+ RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(a));
+ RBIMPL_ASSERT_OR_ASSUME(!SPECIAL_CONST_P(b));
+#endif
+ GC_ASSERT(!during_gc);
GC_ASSERT(RB_BUILTIN_TYPE(a) != T_NONE);
GC_ASSERT(RB_BUILTIN_TYPE(a) != T_MOVED);
GC_ASSERT(RB_BUILTIN_TYPE(a) != T_ZOMBIE);
@@ -6099,7 +6312,7 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
else {
bool retry = false;
/* slow path */
- int lev = rb_gc_vm_lock_no_barrier();
+ int lev = RB_GC_VM_LOCK_NO_BARRIER();
{
if (is_incremental_marking(objspace)) {
gc_writebarrier_incremental(a, b, objspace);
@@ -6108,7 +6321,7 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
retry = true;
}
}
- rb_gc_vm_unlock_no_barrier(lev);
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
if (retry) goto retry;
}
@@ -6127,7 +6340,7 @@ rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", rb_obj_info(obj),
RVALUE_REMEMBERED(objspace, obj) ? " (already remembered)" : "");
- unsigned int lev = rb_gc_vm_lock_no_barrier();
+ unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
{
if (RVALUE_OLD_P(objspace, obj)) {
gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", rb_obj_info(obj));
@@ -6149,7 +6362,7 @@ rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
}
- rb_gc_vm_unlock_no_barrier(lev);
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
}
}
@@ -6177,19 +6390,29 @@ rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", rb_obj_info(obj));
- if (is_incremental_marking(objspace)) {
- if (RVALUE_BLACK_P(objspace, obj)) {
- gc_grey(objspace, obj);
- }
- }
- else {
- if (RVALUE_OLD_P(objspace, obj)) {
- rgengc_remember(objspace, obj);
+ if (is_incremental_marking(objspace) || RVALUE_OLD_P(objspace, obj)) {
+ int lev = RB_GC_VM_LOCK_NO_BARRIER();
+ {
+ if (is_incremental_marking(objspace)) {
+ if (RVALUE_BLACK_P(objspace, obj)) {
+ gc_grey(objspace, obj);
+ }
+ }
+ else if (RVALUE_OLD_P(objspace, obj)) {
+ rgengc_remember(objspace, obj);
+ }
}
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
}
}
-#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 7
+struct rb_gc_object_metadata_names {
+ // Must be ID only
+ ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking,
+ ID_marked, ID_pinned, ID_remembered, ID_object_id, ID_shareable;
+};
+
+#define RB_GC_OBJECT_METADATA_ENTRY_COUNT (sizeof(struct rb_gc_object_metadata_names) / sizeof(ID))
static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1];
struct rb_gc_object_metadata_entry *
@@ -6197,10 +6420,10 @@ rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
{
rb_objspace_t *objspace = objspace_ptr;
size_t n = 0;
- static ID ID_wb_protected, ID_age, ID_old, ID_uncollectible, ID_marking, ID_marked, ID_pinned;
+ static struct rb_gc_object_metadata_names names;
- if (!ID_marked) {
-#define I(s) ID_##s = rb_intern(#s);
+ if (!names.ID_marked) {
+#define I(s) names.ID_##s = rb_intern(#s)
I(wb_protected);
I(age);
I(old);
@@ -6208,12 +6431,15 @@ rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
I(marking);
I(marked);
I(pinned);
+ I(remembered);
+ I(object_id);
+ I(shareable);
#undef I
}
#define SET_ENTRY(na, v) do { \
GC_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
- object_metadata_entries[n].name = ID_##na; \
+ object_metadata_entries[n].name = names.ID_##na; \
object_metadata_entries[n].val = v; \
n++; \
} while (0)
@@ -6225,6 +6451,9 @@ rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
if (RVALUE_MARKING(objspace, obj)) SET_ENTRY(marking, Qtrue);
if (RVALUE_MARKED(objspace, obj)) SET_ENTRY(marked, Qtrue);
if (RVALUE_PINNED(objspace, obj)) SET_ENTRY(pinned, Qtrue);
+ if (RVALUE_REMEMBERED(objspace, obj)) SET_ENTRY(remembered, Qtrue);
+ if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
+ if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue);
object_metadata_entries[n].name = 0;
object_metadata_entries[n].val = 0;
@@ -6249,7 +6478,6 @@ rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache)
rb_objspace_t *objspace = objspace_ptr;
objspace->live_ractor_cache_count--;
-
gc_ractor_newobj_cache_clear(cache, NULL);
free(cache);
}
@@ -6259,7 +6487,7 @@ heap_ready_to_gc(rb_objspace_t *objspace, rb_heap_t *heap)
{
if (!heap->free_pages) {
if (!heap_page_allocate_and_initialize(objspace, heap)) {
- objspace->heap_pages.allocatable_slots = 1;
+ objspace->heap_pages.allocatable_bytes = HEAP_PAGE_SIZE;
heap_page_allocate_and_initialize(objspace, heap);
}
}
@@ -6285,11 +6513,14 @@ gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
{
gc_prof_set_malloc_info(objspace);
{
- size_t inc = RUBY_ATOMIC_SIZE_EXCHANGE(malloc_increase, 0);
+ int64_t inc = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.counters);
size_t old_limit = malloc_limit;
- if (inc > malloc_limit) {
- malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
+ /* A net-negative `inc` (more freed than malloc'd since last GC) is
+ * treated the same as "allocated less than malloc_limit".
+ * This matches what we were doing pre-monotonic counters, but is it right? */
+ if (inc > 0 && (size_t)inc > malloc_limit) {
+ malloc_limit = (size_t)((size_t)inc * gc_params.malloc_limit_growth_factor);
if (malloc_limit > gc_params.malloc_limit_max) {
malloc_limit = gc_params.malloc_limit_max;
}
@@ -6316,7 +6547,11 @@ gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
/* reset oldmalloc info */
#if RGENGC_ESTIMATE_OLDMALLOC
if (!full_mark) {
- if (objspace->rgengc.oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
+ /* Don't snapshot on minor GC: oldmalloc_increase is meant to
+ * accumulate across minor GCs and only reset at major GC. */
+ int64_t oldmalloc_increase = gc_malloc_counters_increase(objspace, &objspace->malloc_counters.oldcounters);
+ if (oldmalloc_increase > 0 &&
+ (uint64_t)oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
objspace->rgengc.oldmalloc_increase_limit =
(size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
@@ -6326,17 +6561,14 @@ gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
}
}
- if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRIuSIZE"\t%"PRIuSIZE"\t%"PRIdSIZE"\n",
+ if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRId64"\t%"PRIuSIZE"\t%"PRIdSIZE"\n",
rb_gc_count(),
gc_needs_major_flags,
- objspace->rgengc.oldmalloc_increase,
+ oldmalloc_increase,
objspace->rgengc.oldmalloc_increase_limit,
gc_params.oldmalloc_limit_max);
}
else {
- /* major GC */
- objspace->rgengc.oldmalloc_increase = 0;
-
if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
objspace->rgengc.oldmalloc_increase_limit =
(size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
@@ -6353,7 +6585,7 @@ garbage_collect(rb_objspace_t *objspace, unsigned int reason)
{
int ret;
- int lev = rb_gc_vm_lock();
+ int lev = RB_GC_VM_LOCK();
{
#if GC_PROFILE_MORE_DETAIL
objspace->profile.prepare_time = getrusage_time();
@@ -6367,7 +6599,7 @@ garbage_collect(rb_objspace_t *objspace, unsigned int reason)
ret = gc_start(objspace, reason);
}
- rb_gc_vm_unlock(lev);
+ RB_GC_VM_UNLOCK(lev);
return ret;
}
@@ -6377,19 +6609,21 @@ gc_start(rb_objspace_t *objspace, unsigned int reason)
{
unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
- /* reason may be clobbered, later, so keep set immediate_sweep here */
- objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
-
if (!rb_darray_size(objspace->heap_pages.sorted)) return TRUE; /* heap is not ready */
if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
- GC_ASSERT(gc_mode(objspace) == gc_mode_none);
+ rb_gc_initialize_vm_context(&objspace->vm_context);
+
+ GC_ASSERT(gc_mode(objspace) == gc_mode_none, "gc_mode is %s\n", gc_mode_name(gc_mode(objspace)));
GC_ASSERT(!is_lazy_sweeping(objspace));
GC_ASSERT(!is_incremental_marking(objspace));
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_start, &lock_lev);
+ /* reason may be clobbered, later, so keep set immediate_sweep here */
+ objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
+
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
@@ -6449,7 +6683,6 @@ gc_start(rb_objspace_t *objspace, unsigned int reason)
reason,
do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
-#if USE_DEBUG_COUNTER
RB_DEBUG_COUNTER_INC(gc_count);
if (reason & GPR_FLAG_MAJOR_MASK) {
@@ -6468,14 +6701,13 @@ gc_start(rb_objspace_t *objspace, unsigned int reason)
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
}
-#endif
objspace->profile.count++;
objspace->profile.latest_gc_info = reason;
objspace->profile.total_allocated_objects_at_gc_start = total_allocated_objects(objspace);
objspace->profile.heap_used_at_gc_start = rb_darray_size(objspace->heap_pages.sorted);
+ objspace->profile.heap_total_slots_at_gc_start = objspace_available_slots(objspace);
objspace->profile.weak_references_count = 0;
- objspace->profile.retained_weak_references_count = 0;
gc_prof_setup_new_record(objspace, reason);
gc_reset_malloc_info(objspace, do_full_mark);
@@ -6653,12 +6885,10 @@ gc_clock_end(struct timespec *ts)
static inline void
gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
{
- *lock_lev = rb_gc_vm_lock();
+ *lock_lev = RB_GC_VM_LOCK();
switch (event) {
case gc_enter_event_rest:
- if (!is_marking(objspace)) break;
- // fall through
case gc_enter_event_start:
case gc_enter_event_continue:
// stop other ractors
@@ -6692,7 +6922,7 @@ gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_l
gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
during_gc = FALSE;
- rb_gc_vm_unlock(*lock_lev);
+ RB_GC_VM_UNLOCK(*lock_lev);
}
#ifndef MEASURE_GC
@@ -6704,9 +6934,13 @@ gc_marking_enter(rb_objspace_t *objspace)
{
GC_ASSERT(during_gc != 0);
+ gc_prof_mark_timer_start(objspace);
+
if (MEASURE_GC) {
gc_clock_start(&objspace->profile.marking_start_time);
}
+
+ rb_gc_initialize_vm_context(&objspace->vm_context);
}
static void
@@ -6717,6 +6951,8 @@ gc_marking_exit(rb_objspace_t *objspace)
if (MEASURE_GC) {
objspace->profile.marking_time_ns += gc_clock_end(&objspace->profile.marking_start_time);
}
+
+ gc_prof_mark_timer_stop(objspace);
}
static void
@@ -6831,7 +7067,7 @@ rb_gc_impl_prepare_heap(void *objspace_ptr)
rb_objspace_t *objspace = objspace_ptr;
size_t orig_total_slots = objspace_available_slots(objspace);
- size_t orig_allocatable_slots = objspace->heap_pages.allocatable_slots;
+ size_t orig_allocatable_bytes = objspace->heap_pages.allocatable_bytes;
rb_gc_impl_each_objects(objspace, gc_set_candidate_object_i, objspace_ptr);
@@ -6841,14 +7077,16 @@ rb_gc_impl_prepare_heap(void *objspace_ptr)
rb_gc_impl_start(objspace, true, true, true, true);
gc_params.heap_free_slots_max_ratio = orig_max_free_slots;
- objspace->heap_pages.allocatable_slots = 0;
+ objspace->heap_pages.allocatable_bytes = 0;
+ heap_pages_freeable_pages = objspace->empty_pages_count;
heap_pages_free_unused_pages(objspace_ptr);
+ GC_ASSERT(heap_pages_freeable_pages == 0);
GC_ASSERT(objspace->empty_pages_count == 0);
- objspace->heap_pages.allocatable_slots = orig_allocatable_slots;
+ objspace->heap_pages.allocatable_bytes = orig_allocatable_bytes;
size_t total_slots = objspace_available_slots(objspace);
if (orig_total_slots > total_slots) {
- objspace->heap_pages.allocatable_slots += orig_total_slots - total_slots;
+ objspace->heap_pages.allocatable_bytes += (orig_total_slots - total_slots) * heaps[0].slot_size;
}
#if defined(HAVE_MALLOC_TRIM) && !defined(RUBY_ALTERNATIVE_MALLOC_HEADER)
@@ -6867,12 +7105,6 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
case T_ZOMBIE:
return FALSE;
case T_SYMBOL:
- // TODO: restore original behavior
- // if (RSYMBOL(obj)->id & ~ID_SCOPE_MASK) {
- // return FALSE;
- // }
- return false;
- /* fall through */
case T_STRING:
case T_OBJECT:
case T_FLOAT:
@@ -6891,7 +7123,7 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
case T_RATIONAL:
case T_NODE:
case T_CLASS:
- if (FL_TEST(obj, FL_FINALIZE)) {
+ if (FL_TEST_RAW(obj, FL_FINALIZE)) {
/* The finalizer table is a numtable. It looks up objects by address.
* We can't mark the keys in the finalizer table because that would
* prevent the objects from being collected. This check prevents
@@ -6917,8 +7149,11 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
void rb_mv_generic_ivar(VALUE src, VALUE dst);
static VALUE
-gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, size_t slot_size)
+gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, struct heap_page *src_page, struct heap_page *dest_page)
{
+ size_t src_slot_size = src_page->slot_size;
+ size_t slot_size = dest_page->slot_size;
+
int marked;
int wb_unprotected;
int uncollectible;
@@ -6944,30 +7179,13 @@ gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, si
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(src), src);
CLEAR_IN_BITMAP(GET_HEAP_PAGE(src)->remembered_bits, src);
- if (FL_TEST(src, FL_SEEN_OBJ_ID)) {
- /* If the source object's object_id has been seen, we need to update
- * the object to object id mapping. */
- st_data_t srcid = (st_data_t)src, id;
-
- gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest);
- /* Resizing the st table could cause a malloc */
- DURING_GC_COULD_MALLOC_REGION_START();
- {
- if (!st_delete(objspace->obj_to_id_tbl, &srcid, &id)) {
- rb_bug("gc_move: object ID seen, but not in mapping table: %s", rb_obj_info((VALUE)src));
- }
-
- st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id);
- }
- DURING_GC_COULD_MALLOC_REGION_END();
- }
- else {
- GC_ASSERT(!st_lookup(objspace->obj_to_id_tbl, (st_data_t)src, NULL));
- }
-
/* Move the object */
memcpy((void *)dest, (void *)src, MIN(src_slot_size, slot_size));
+ if (src_slot_size != slot_size && RB_TYPE_P(src, T_OBJECT)) {
+ rb_gc_obj_changed_pool(dest, dest_page->heap - heaps);
+ }
+
if (RVALUE_OVERHEAD > 0) {
void *dest_overhead = (void *)(((uintptr_t)dest) + slot_size - RVALUE_OVERHEAD);
void *src_overhead = (void *)(((uintptr_t)src) + src_slot_size - RVALUE_OVERHEAD);
@@ -6976,7 +7194,7 @@ gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, si
}
memset((void *)src, 0, src_slot_size);
- RVALUE_AGE_RESET(src);
+ RVALUE_AGE_SET_BITMAP(src, 0);
/* Set bits for object in new location */
if (remembered) {
@@ -7083,6 +7301,12 @@ gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func co
}
#endif
+void
+rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
+{
+ /* no-op */
+}
+
bool
rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
{
@@ -7150,6 +7374,8 @@ gc_update_references(rb_objspace_t *objspace)
{
objspace->flags.during_reference_updating = true;
+ rb_gc_before_updating_jit_code();
+
struct heap_page *page = NULL;
for (int i = 0; i < HEAP_COUNT; i++) {
@@ -7169,8 +7395,7 @@ gc_update_references(rb_objspace_t *objspace)
}
}
}
- gc_ref_update_table_values_only(objspace->obj_to_id_tbl);
- gc_update_table_refs(objspace->id_to_obj_tbl);
+
gc_update_table_refs(finalizer_table);
rb_gc_update_vm_references((void *)objspace);
@@ -7185,6 +7410,8 @@ gc_update_references(rb_objspace_t *objspace)
);
}
+ rb_gc_after_updating_jit_code();
+
objspace->flags.during_reference_updating = false;
}
@@ -7313,7 +7540,7 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned
#endif
static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
static VALUE sym_none, sym_marking, sym_sweeping;
- static VALUE sym_weak_references_count, sym_retained_weak_references_count;
+ static VALUE sym_weak_references_count;
VALUE hash = Qnil, key = Qnil;
VALUE major_by, need_major_by;
unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
@@ -7355,7 +7582,6 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned
S(sweeping);
S(weak_references_count);
- S(retained_weak_references_count);
#undef S
}
@@ -7408,7 +7634,6 @@ gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned
}
SET(weak_references_count, LONG2FIX(objspace->profile.weak_references_count));
- SET(retained_weak_references_count, LONG2FIX(objspace->profile.retained_weak_references_count));
#undef SET
if (!NIL_P(key)) {
@@ -7435,7 +7660,7 @@ enum gc_stat_sym {
gc_stat_sym_sweeping_time,
gc_stat_sym_heap_allocated_pages,
gc_stat_sym_heap_empty_pages,
- gc_stat_sym_heap_allocatable_slots,
+ gc_stat_sym_heap_allocatable_bytes,
gc_stat_sym_heap_available_slots,
gc_stat_sym_heap_live_slots,
gc_stat_sym_heap_free_slots,
@@ -7446,6 +7671,8 @@ enum gc_stat_sym {
gc_stat_sym_total_freed_pages,
gc_stat_sym_total_allocated_objects,
gc_stat_sym_total_freed_objects,
+ gc_stat_sym_total_malloc_bytes,
+ gc_stat_sym_total_free_bytes,
gc_stat_sym_malloc_increase_bytes,
gc_stat_sym_malloc_increase_bytes_limit,
gc_stat_sym_minor_gc_count,
@@ -7461,7 +7688,6 @@ enum gc_stat_sym {
gc_stat_sym_oldmalloc_increase_bytes,
gc_stat_sym_oldmalloc_increase_bytes_limit,
#endif
- gc_stat_sym_weak_references_count,
#if RGENGC_PROFILE
gc_stat_sym_total_generated_normal_object_count,
gc_stat_sym_total_generated_shady_object_count,
@@ -7486,7 +7712,7 @@ setup_gc_stat_symbols(void)
S(sweeping_time),
S(heap_allocated_pages);
S(heap_empty_pages);
- S(heap_allocatable_slots);
+ S(heap_allocatable_bytes);
S(heap_available_slots);
S(heap_live_slots);
S(heap_free_slots);
@@ -7497,6 +7723,8 @@ setup_gc_stat_symbols(void)
S(total_freed_pages);
S(total_allocated_objects);
S(total_freed_objects);
+ S(total_malloc_bytes);
+ S(total_free_bytes);
S(malloc_increase_bytes);
S(malloc_increase_bytes_limit);
S(minor_gc_count);
@@ -7512,7 +7740,6 @@ setup_gc_stat_symbols(void)
S(oldmalloc_increase_bytes);
S(oldmalloc_increase_bytes_limit);
#endif
- S(weak_references_count);
#if RGENGC_PROFILE
S(total_generated_normal_object_count);
S(total_generated_shady_object_count);
@@ -7531,6 +7758,8 @@ ns_to_ms(uint64_t ns)
return ns / (1000 * 1000);
}
+static void malloc_increase_local_flush(rb_objspace_t *objspace);
+
VALUE
rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
{
@@ -7539,6 +7768,9 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
setup_gc_stat_symbols();
+ ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache());
+ malloc_increase_local_flush(objspace);
+
if (RB_TYPE_P(hash_or_sym, T_HASH)) {
hash = hash_or_sym;
}
@@ -7554,27 +7786,32 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
return SIZET2NUM(attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
+#define SET64(name, attr) \
+ if (key == gc_stat_symbols[gc_stat_sym_##name]) \
+ return ULL2NUM(attr); \
+ else if (hash != Qnil) \
+ rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], ULL2NUM(attr));
SET(count, objspace->profile.count);
SET(time, (size_t)ns_to_ms(objspace->profile.marking_time_ns + objspace->profile.sweeping_time_ns)); // TODO: UINT64T2NUM
SET(marking_time, (size_t)ns_to_ms(objspace->profile.marking_time_ns));
SET(sweeping_time, (size_t)ns_to_ms(objspace->profile.sweeping_time_ns));
- /* implementation dependent counters */
+ {
+ uint64_t total_malloc = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.malloc);
+ uint64_t total_free = (uint64_t)gc_counter_load_relaxed(&objspace->malloc_counters.counters.free);
+ SET64(total_malloc_bytes, total_malloc);
+ SET64(total_free_bytes, total_free);
+ }
+
+ /* implementation dependent counters (small / fixnum-safe) */
SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted));
SET(heap_empty_pages, objspace->empty_pages_count)
- SET(heap_allocatable_slots, objspace->heap_pages.allocatable_slots);
- SET(heap_available_slots, objspace_available_slots(objspace));
- SET(heap_live_slots, objspace_live_slots(objspace));
- SET(heap_free_slots, objspace_free_slots(objspace));
- SET(heap_final_slots, total_final_slots_count(objspace));
- SET(heap_marked_slots, objspace->marked_slots);
+ SET(heap_allocatable_bytes, objspace->heap_pages.allocatable_bytes);
SET(heap_eden_pages, heap_eden_total_pages(objspace));
SET(total_allocated_pages, objspace->heap_pages.allocated_pages);
SET(total_freed_pages, objspace->heap_pages.freed_pages);
- SET(total_allocated_objects, total_allocated_objects(objspace));
- SET(total_freed_objects, total_freed_objects(objspace));
- SET(malloc_increase_bytes, malloc_increase);
+ SET(malloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.counters));
SET(malloc_increase_bytes_limit, malloc_limit);
SET(minor_gc_count, objspace->profile.minor_gc_count);
SET(major_gc_count, objspace->profile.major_gc_count);
@@ -7586,10 +7823,19 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
SET(old_objects, objspace->rgengc.old_objects);
SET(old_objects_limit, objspace->rgengc.old_objects_limit);
#if RGENGC_ESTIMATE_OLDMALLOC
- SET(oldmalloc_increase_bytes, objspace->rgengc.oldmalloc_increase);
+ SET(oldmalloc_increase_bytes, gc_malloc_counters_increase_unsigned(objspace, &objspace->malloc_counters.oldcounters));
SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
#endif
+ ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache());
+ SET(total_allocated_objects, total_allocated_objects(objspace));
+ SET(total_freed_objects, total_freed_objects(objspace));
+ SET(heap_available_slots, objspace_available_slots(objspace));
+ SET(heap_live_slots, objspace_live_slots(objspace));
+ SET(heap_free_slots, objspace_free_slots(objspace));
+ SET(heap_final_slots, total_final_slots_count(objspace));
+ SET(heap_marked_slots, objspace->marked_slots);
+
#if RGENGC_PROFILE
SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
@@ -7599,6 +7845,7 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
#endif /* RGENGC_PROFILE */
#undef SET
+#undef SET64
if (!NIL_P(key)) {
// Matched key should return above
@@ -7621,11 +7868,15 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
enum gc_stat_heap_sym {
gc_stat_heap_sym_slot_size,
+ gc_stat_heap_sym_heap_live_slots,
+ gc_stat_heap_sym_heap_free_slots,
+ gc_stat_heap_sym_heap_final_slots,
gc_stat_heap_sym_heap_eden_pages,
gc_stat_heap_sym_heap_eden_slots,
gc_stat_heap_sym_total_allocated_pages,
gc_stat_heap_sym_force_major_gc_count,
gc_stat_heap_sym_force_incremental_marking_finish_count,
+ gc_stat_heap_sym_heap_allocatable_slots,
gc_stat_heap_sym_total_allocated_objects,
gc_stat_heap_sym_total_freed_objects,
gc_stat_heap_sym_last
@@ -7639,8 +7890,12 @@ setup_gc_stat_heap_symbols(void)
if (gc_stat_heap_symbols[0] == 0) {
#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
S(slot_size);
+ S(heap_live_slots);
+ S(heap_free_slots);
+ S(heap_final_slots);
S(heap_eden_pages);
S(heap_eden_slots);
+ S(heap_allocatable_slots);
S(total_allocated_pages);
S(force_major_gc_count);
S(force_incremental_marking_finish_count);
@@ -7651,7 +7906,7 @@ setup_gc_stat_heap_symbols(void)
}
static VALUE
-stat_one_heap(rb_heap_t *heap, VALUE hash, VALUE key)
+stat_one_heap(rb_objspace_t *objspace, rb_heap_t *heap, VALUE hash, VALUE key)
{
#define SET(name, attr) \
if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
@@ -7660,8 +7915,12 @@ stat_one_heap(rb_heap_t *heap, VALUE hash, VALUE key)
rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
SET(slot_size, heap->slot_size);
+ SET(heap_live_slots, heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count);
+ SET(heap_free_slots, heap->total_slots - (heap->total_allocated_objects - heap->total_freed_objects));
+ SET(heap_final_slots, heap->final_slots_count);
SET(heap_eden_pages, heap->total_pages);
SET(heap_eden_slots, heap->total_slots);
+ SET(heap_allocatable_slots, objspace->heap_pages.allocatable_bytes / heap->slot_size);
SET(total_allocated_pages, heap->total_allocated_pages);
SET(force_major_gc_count, heap->force_major_gc_count);
SET(force_incremental_marking_finish_count, heap->force_incremental_marking_finish_count);
@@ -7682,6 +7941,8 @@ rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
{
rb_objspace_t *objspace = objspace_ptr;
+ ractor_cache_flush_count(objspace, rb_gc_get_ractor_newobj_cache());
+
setup_gc_stat_heap_symbols();
if (NIL_P(heap_name)) {
@@ -7696,7 +7957,7 @@ rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
rb_hash_aset(hash_or_sym, INT2FIX(i), hash);
}
- stat_one_heap(&heaps[i], hash, Qnil);
+ stat_one_heap(objspace, &heaps[i], hash, Qnil);
}
}
else if (FIXNUM_P(heap_name)) {
@@ -7707,10 +7968,10 @@ rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
}
if (SYMBOL_P(hash_or_sym)) {
- return stat_one_heap(&heaps[heap_idx], Qnil, hash_or_sym);
+ return stat_one_heap(objspace, &heaps[heap_idx], Qnil, hash_or_sym);
}
else if (RB_TYPE_P(hash_or_sym, T_HASH)) {
- return stat_one_heap(&heaps[heap_idx], hash_or_sym, Qnil);
+ return stat_one_heap(objspace, &heaps[heap_idx], hash_or_sym, Qnil);
}
else {
rb_bug("non-hash or symbol given");
@@ -7745,7 +8006,7 @@ rb_gc_impl_config_get(void *objspace_ptr)
}
static int
-gc_config_set_key(st_data_t key, st_data_t value, st_data_t data)
+gc_config_set_key(VALUE key, VALUE value, VALUE data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
if (rb_sym2id(key) == rb_intern("rgengc_allow_full_mark")) {
@@ -7764,7 +8025,7 @@ rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
rb_raise(rb_eArgError, "expected keyword arguments");
}
- rb_hash_stlike_foreach(hash, gc_config_set_key, (st_data_t)objspace);
+ rb_hash_foreach(hash, gc_config_set_key, (st_data_t)objspace);
}
VALUE
@@ -7892,8 +8153,8 @@ get_envparam_double(const char *name, double *default_value, double lower_bound,
* * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
* - Allocate slots by this factor.
* - (next slots number) = (current slots number) * (this factor)
- * * RUBY_GC_HEAP_GROWTH_MAX_SLOTS (new from 2.1)
- * - Allocation rate is limited to this number of slots.
+ * * RUBY_GC_HEAP_GROWTH_MAX_BYTES (was RUBY_GC_HEAP_GROWTH_MAX_SLOTS)
+ * - Allocation rate is limited to this number of bytes.
* * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
* - Allocate additional pages when the number of free slots is
* lower than the value (total_slots * (this ratio)).
@@ -7912,7 +8173,7 @@ get_envparam_double(const char *name, double *default_value, double lower_bound,
*
* * obsolete
* * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
- * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1)
+ * * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1) -> RUBY_GC_HEAP_INIT_BYTES
*
* * RUBY_GC_MALLOC_LIMIT
* * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
@@ -7932,15 +8193,10 @@ rb_gc_impl_set_params(void *objspace_ptr)
/* ok */
}
- for (int i = 0; i < HEAP_COUNT; i++) {
- char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)];
- snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i);
-
- get_envparam_size(env_key, &gc_params.heap_init_slots[i], 0);
- }
+ get_envparam_size("RUBY_GC_HEAP_INIT_BYTES", &gc_params.heap_init_bytes, 0);
get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
- get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0);
+ get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_BYTES", &gc_params.growth_max_bytes, 0);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
0.0, 1.0, FALSE);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
@@ -8014,8 +8270,53 @@ objspace_malloc_gc_stress(rb_objspace_t *objspace)
}
}
+static void
+malloc_increase_commit(rb_objspace_t *objspace, size_t new_size, size_t old_size)
+{
+ if (new_size > old_size) {
+ size_t delta = new_size - old_size;
+ MALLOC_COUNTERS_LOCK(objspace);
+ gc_counter_add(&objspace->malloc_counters.counters.malloc, delta);
+#if RGENGC_ESTIMATE_OLDMALLOC
+ gc_counter_add(&objspace->malloc_counters.oldcounters.malloc, delta);
+#endif
+ MALLOC_COUNTERS_UNLOCK(objspace);
+ }
+ else if (old_size > new_size) {
+ size_t delta = old_size - new_size;
+ MALLOC_COUNTERS_LOCK(objspace);
+ gc_counter_add(&objspace->malloc_counters.counters.free, delta);
+#if RGENGC_ESTIMATE_OLDMALLOC
+ gc_counter_add(&objspace->malloc_counters.oldcounters.free, delta);
+#endif
+ MALLOC_COUNTERS_UNLOCK(objspace);
+ }
+}
+
+#if USE_MALLOC_INCREASE_LOCAL
+static void
+malloc_increase_local_flush(rb_objspace_t *objspace)
+{
+ int delta = malloc_increase_local;
+ if (delta == 0) return;
+
+ malloc_increase_local = 0;
+ if (delta > 0) {
+ malloc_increase_commit(objspace, (size_t)delta, 0);
+ }
+ else {
+ malloc_increase_commit(objspace, 0, (size_t)(-delta));
+ }
+}
+#else
+static void
+malloc_increase_local_flush(rb_objspace_t *objspace)
+{
+}
+#endif
+
static inline bool
-objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
+objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
{
if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n",
mem,
@@ -8027,22 +8328,27 @@ objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_s
}
static bool
-objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
+objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type, bool gc_allowed)
{
- if (new_size > old_size) {
- RUBY_ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size);
-#if RGENGC_ESTIMATE_OLDMALLOC
- RUBY_ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size);
-#endif
+#if USE_MALLOC_INCREASE_LOCAL
+ if (new_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD &&
+ old_size < GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
+ malloc_increase_local += (int)new_size - (int)old_size;
+
+ if (malloc_increase_local >= GC_MALLOC_INCREASE_LOCAL_THRESHOLD ||
+ malloc_increase_local <= -GC_MALLOC_INCREASE_LOCAL_THRESHOLD) {
+ malloc_increase_local_flush(objspace);
+ }
}
else {
- atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
-#if RGENGC_ESTIMATE_OLDMALLOC
- atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size);
-#endif
+ malloc_increase_local_flush(objspace);
+ malloc_increase_commit(objspace, new_size, old_size);
}
+#else
+ malloc_increase_commit(objspace, new_size, old_size);
+#endif
- if (type == MEMOP_TYPE_MALLOC) {
+ if (type == MEMOP_TYPE_MALLOC && gc_allowed) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) {
if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) {
@@ -8124,10 +8430,10 @@ malloc_during_gc_p(rb_objspace_t *objspace)
}
static inline void *
-objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
+objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size, bool gc_allowed)
{
size = objspace_malloc_size(objspace, mem, size);
- objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC) {}
+ objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC, gc_allowed) {}
#if CALC_EXACT_MALLOC_SIZE
{
@@ -8159,10 +8465,10 @@ objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
GPR_FLAG_MALLOC; \
objspace_malloc_gc_stress(objspace); \
\
- if (RB_LIKELY((expr))) { \
+ if (RB_LIKELY((expr))) { \
/* Success on 1st try */ \
} \
- else if (!garbage_collect_with_gvl(objspace, gpr)) { \
+ else if (gc_allowed && !garbage_collect_with_gvl(objspace, gpr)) { \
/* @shyouhei thinks this doesn't happen */ \
GC_MEMERROR("TRY_WITH_GC: could not GC"); \
} \
@@ -8200,12 +8506,21 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
}
#if CALC_EXACT_MALLOC_SIZE
struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
+#if VERIFY_FREE_SIZE
+ if (!info->size) {
+ rb_bug("buffer %p has no recorded size. Was it allocated with ruby_mimalloc? If so it should be freed with ruby_mimfree", ptr);
+ }
+
+ if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
+ rb_bug("buffer %p freed with old_size=%zu, but was allocated with size=%zu", ptr, old_size, info->size - sizeof(struct malloc_obj_info));
+ }
+#endif
ptr = info;
old_size = info->size;
#endif
old_size = objspace_malloc_size(objspace, ptr, old_size);
- objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE) {
+ objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE, true) {
free(ptr);
ptr = NULL;
RB_DEBUG_COUNTER_INC(heap_xfree);
@@ -8213,7 +8528,7 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
}
void *
-rb_gc_impl_malloc(void *objspace_ptr, size_t size)
+rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
rb_objspace_t *objspace = objspace_ptr;
check_malloc_not_in_gc(objspace, "malloc");
@@ -8224,11 +8539,11 @@ rb_gc_impl_malloc(void *objspace_ptr, size_t size)
TRY_WITH_GC(size, mem = malloc(size));
RB_DEBUG_COUNTER_INC(heap_xmalloc);
if (!mem) return mem;
- return objspace_malloc_fixup(objspace, mem, size);
+ return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
}
void *
-rb_gc_impl_calloc(void *objspace_ptr, size_t size)
+rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
rb_objspace_t *objspace = objspace_ptr;
@@ -8244,11 +8559,11 @@ rb_gc_impl_calloc(void *objspace_ptr, size_t size)
size = objspace_malloc_prepare(objspace, size);
TRY_WITH_GC(size, mem = calloc1(size));
if (!mem) return mem;
- return objspace_malloc_fixup(objspace, mem, size);
+ return objspace_malloc_fixup(objspace, mem, size, gc_allowed);
}
void *
-rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size)
+rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
{
rb_objspace_t *objspace = objspace_ptr;
@@ -8256,7 +8571,7 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si
void *mem;
- if (!ptr) return rb_gc_impl_malloc(objspace, new_size);
+ if (!ptr) return rb_gc_impl_malloc(objspace, new_size, gc_allowed);
/*
* The behavior of realloc(ptr, 0) is implementation defined.
@@ -8264,7 +8579,7 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si
* see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
*/
if (new_size == 0) {
- if ((mem = rb_gc_impl_malloc(objspace, 0)) != NULL) {
+ if ((mem = rb_gc_impl_malloc(objspace, 0, gc_allowed)) != NULL) {
/*
* - OpenBSD's malloc(3) man page says that when 0 is passed, it
* returns a non-NULL pointer to an access-protected memory page.
@@ -8306,6 +8621,11 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si
struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
new_size += sizeof(struct malloc_obj_info);
ptr = info;
+#if VERIFY_FREE_SIZE
+ if (old_size && (old_size + sizeof(struct malloc_obj_info)) != info->size) {
+ rb_bug("buffer %p realloced with old_size=%zu, but was allocated with size=%zu", ptr, old_size, info->size - sizeof(struct malloc_obj_info));
+ }
+#endif
old_size = info->size;
}
#endif
@@ -8323,7 +8643,7 @@ rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_si
}
#endif
- objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
+ objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC, gc_allowed);
RB_DEBUG_COUNTER_INC(heap_xrealloc);
return mem;
@@ -8335,10 +8655,10 @@ rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff)
rb_objspace_t *objspace = objspace_ptr;
if (diff > 0) {
- objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
+ objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC, true);
}
else if (diff < 0) {
- objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
+ objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC, true);
}
}
@@ -8577,18 +8897,29 @@ gc_prof_set_heap_info(rb_objspace_t *objspace)
{
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
- size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace);
- size_t total = objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT;
+
+ /* Sum across all size pools since each has a different slot size. */
+ size_t total = 0;
+ size_t use_size = 0;
+ size_t total_size = 0;
+ for (int i = 0; i < HEAP_COUNT; i++) {
+ rb_heap_t *heap = &heaps[i];
+ size_t heap_live = heap->total_allocated_objects - heap->total_freed_objects - heap->final_slots_count;
+ total += heap->total_slots;
+ use_size += heap_live * heap->slot_size;
+ total_size += heap->total_slots * heap->slot_size;
+ }
#if GC_PROFILE_MORE_DETAIL
+ size_t live = objspace->profile.total_allocated_objects_at_gc_start - total_freed_objects(objspace);
record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
record->heap_live_objects = live;
record->heap_free_objects = total - live;
#endif
record->heap_total_objects = total;
- record->heap_use_size = live * BASE_SLOT_SIZE;
- record->heap_total_size = total * BASE_SLOT_SIZE;
+ record->heap_use_size = use_size;
+ record->heap_total_size = total_size;
}
}
@@ -8955,6 +9286,12 @@ gc_profile_disable(VALUE _)
return Qnil;
}
+void
+rb_gc_verify_internal_consistency(void)
+{
+ gc_verify_internal_consistency(rb_gc_get_objspace());
+}
+
/*
* call-seq:
* GC.verify_internal_consistency -> nil
@@ -8968,7 +9305,7 @@ gc_profile_disable(VALUE _)
static VALUE
gc_verify_internal_consistency_m(VALUE dummy)
{
- gc_verify_internal_consistency(rb_gc_get_objspace());
+ rb_gc_verify_internal_consistency();
return Qnil;
}
@@ -9189,7 +9526,7 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self)
/* Clear the heap. */
rb_gc_impl_start(objspace, true, true, true, false);
- unsigned int lev = rb_gc_vm_lock();
+ unsigned int lev = RB_GC_VM_LOCK();
{
gc_rest(objspace);
@@ -9225,8 +9562,8 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self)
* Step 2: Now add additional free pages to each size pool sufficient to hold all objects
* that want to be in that size pool, whether moved into it or moved within it
*/
- objspace->heap_pages.allocatable_slots = desired_compaction.required_slots[i];
- while (objspace->heap_pages.allocatable_slots > 0) {
+ objspace->heap_pages.allocatable_bytes = desired_compaction.required_slots[i] * heap->slot_size;
+ while (objspace->heap_pages.allocatable_bytes > 0) {
heap_page_allocate_and_initialize(objspace, heap);
}
/*
@@ -9245,7 +9582,7 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self)
objspace->rcompactor.compare_func = compare_free_slots;
}
}
- rb_gc_vm_unlock(lev);
+ RB_GC_VM_UNLOCK(lev);
rb_gc_impl_start(rb_gc_get_objspace(), true, true, true, true);
@@ -9284,14 +9621,15 @@ rb_gc_impl_objspace_free(void *objspace_ptr)
heap->total_slots = 0;
}
- st_free_table(objspace->id_to_obj_tbl);
- st_free_table(objspace->obj_to_id_tbl);
-
free_stack_chunks(&objspace->mark_stack);
mark_stack_free_cache(&objspace->mark_stack);
rb_darray_free_without_gc(objspace->weak_references);
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+ rb_native_mutex_destroy(&objspace->malloc_counters.lock);
+#endif
+
free(objspace);
}
@@ -9329,9 +9667,31 @@ gc_malloc_allocations(VALUE self)
}
#endif
-void rb_gc_impl_before_fork(void *objspace_ptr) { /* no-op */ }
-void rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) { /* no-op */ }
+void
+rb_gc_impl_before_fork(void *objspace_ptr)
+{
+ rb_objspace_t *objspace = objspace_ptr;
+
+ objspace->fork_vm_lock_lev = RB_GC_VM_LOCK();
+ rb_gc_vm_barrier();
+}
+
+void
+rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
+{
+ rb_objspace_t *objspace = objspace_ptr;
+ RB_GC_VM_UNLOCK(objspace->fork_vm_lock_lev);
+ objspace->fork_vm_lock_lev = 0;
+
+ if (pid == 0) { /* child process */
+ rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, NULL);
+ }
+}
+
+VALUE rb_ident_hash_new_with_size(st_index_t size);
+
+#if GC_DEBUG_STRESS_TO_CLASS
/*
* call-seq:
* GC.add_stress_to_class(class[, ...])
@@ -9381,6 +9741,7 @@ rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self)
return Qnil;
}
+#endif
void *
rb_gc_impl_objspace_alloc(void)
@@ -9399,42 +9760,40 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
objspace->flags.measure_gc = true;
malloc_limit = gc_params.malloc_limit_min;
+#ifdef MALLOC_COUNTERS_NEED_LOCK
+ rb_native_mutex_initialize(&objspace->malloc_counters.lock);
+#endif
objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, objspace);
if (objspace->finalize_deferred_pjob == POSTPONED_JOB_HANDLE_INVALID) {
rb_bug("Could not preregister postponed job for GC");
}
+ /* A standard RVALUE (RBasic + embedded VALUEs + debug overhead) must fit
+ * in at least one pool. In debug builds RVALUE_OVERHEAD can push this
+ * beyond the 48-byte pool into the 64-byte pool, which is fine. */
+ GC_ASSERT(rb_gc_impl_size_allocatable_p(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
+
for (int i = 0; i < HEAP_COUNT; i++) {
rb_heap_t *heap = &heaps[i];
- heap->slot_size = (1 << i) * BASE_SLOT_SIZE;
+ heap->slot_size = pool_slot_sizes[i];
ccan_list_head_init(&heap->pages);
}
+ init_size_to_heap_idx();
+
rb_darray_make_without_gc(&objspace->heap_pages.sorted, 0);
rb_darray_make_without_gc(&objspace->weak_references, 0);
- // TODO: debug why on Windows Ruby crashes on boot when GC is on.
-#ifdef _WIN32
- dont_gc_on();
-#endif
-
#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
/* Need to determine if we can use mmap at runtime. */
heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
#endif
- objspace->next_object_id = OBJ_ID_INITIAL;
- objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
- objspace->obj_to_id_tbl = st_init_numtable();
#if RGENGC_ESTIMATE_OLDMALLOC
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
#endif
- /* Set size pools allocatable pages. */
- for (int i = 0; i < HEAP_COUNT; i++) {
- /* Set the default value of heap_init_slots. */
- gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS;
- }
+ gc_params.heap_init_bytes = GC_HEAP_INIT_BYTES;
init_mark_stack(&objspace->mark_stack);
@@ -9447,9 +9806,14 @@ rb_gc_impl_init(void)
{
VALUE gc_constants = rb_hash_new();
rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
+ /* Minimum slot size that fits a standard RVALUE */
+ size_t rvalue_pool = 0;
+ for (size_t i = 0; i < HEAP_COUNT; i++) {
+ if (pool_slot_sizes[i] >= RVALUE_SLOT_SIZE) { rvalue_pool = pool_slot_sizes[i]; break; }
+ }
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_pool - RVALUE_OVERHEAD));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT));
@@ -9477,10 +9841,10 @@ rb_gc_impl_init(void)
rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
}
- if (GC_DEBUG_STRESS_TO_CLASS) {
- rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
- rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
- }
+#if GC_DEBUG_STRESS_TO_CLASS
+ rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
+ rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
+#endif
/* internal methods */
rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
diff --git a/gc/gc.h b/gc/gc.h
index c2e985252f..d147a3122a 100644
--- a/gc/gc.h
+++ b/gc/gc.h
@@ -10,16 +10,38 @@
* first introduced for [Feature #20470].
*/
#include "ruby/ruby.h"
+#include "ruby/assert.h"
-#if USE_MODULAR_GC
#include "ruby/thread_native.h"
+#ifndef VM_CHECK_MODE
+# define VM_CHECK_MODE RUBY_DEBUG
+#endif
+
+// From ractor_core.h
+#ifndef RACTOR_CHECK_MODE
+# define RACTOR_CHECK_MODE (VM_CHECK_MODE || RUBY_DEBUG) && (SIZEOF_UINT64_T == SIZEOF_VALUE)
+#endif
+
+#if RACTOR_CHECK_MODE
+void rb_ractor_setup_belonging(VALUE obj);
+
+struct rb_gc_obj_suffix {
+ uint32_t _ractor_belonging_id;
+};
+
+# define RB_GC_OBJ_HAS_SUFFIX 1
+# define RB_GC_OBJ_SUFFIX_SIZE (sizeof(struct rb_gc_obj_suffix))
+#else
+# define RB_GC_OBJ_HAS_SUFFIX 0
+# define RB_GC_OBJ_SUFFIX_SIZE 0
+#endif
+
struct rb_gc_vm_context {
rb_nativethread_lock_t lock;
struct rb_execution_context_struct *ec;
};
-#endif
typedef int (*vm_table_foreach_callback_func)(VALUE value, void *data);
typedef int (*vm_table_update_callback_func)(VALUE *value, void *data);
@@ -28,57 +50,88 @@ enum rb_gc_vm_weak_tables {
RB_GC_VM_CI_TABLE,
RB_GC_VM_OVERLOADED_CME_TABLE,
RB_GC_VM_GLOBAL_SYMBOLS_TABLE,
- RB_GC_VM_GENERIC_IV_TABLE,
+ RB_GC_VM_ID2REF_TABLE,
+ RB_GC_VM_GENERIC_FIELDS_TABLE,
RB_GC_VM_FROZEN_STRINGS_TABLE,
RB_GC_VM_WEAK_TABLE_COUNT
};
+#define RB_GC_VM_LOCK() rb_gc_vm_lock(__FILE__, __LINE__)
+#define RB_GC_VM_UNLOCK(lev) rb_gc_vm_unlock(lev, __FILE__, __LINE__)
+#define RB_GC_CR_LOCK() rb_gc_cr_lock(__FILE__, __LINE__)
+#define RB_GC_CR_UNLOCK(lev) rb_gc_cr_unlock(lev, __FILE__, __LINE__)
+#define RB_GC_VM_LOCK_NO_BARRIER() rb_gc_vm_lock_no_barrier(__FILE__, __LINE__)
+#define RB_GC_VM_UNLOCK_NO_BARRIER(lev) rb_gc_vm_unlock_no_barrier(lev, __FILE__, __LINE__)
+
+#if USE_MODULAR_GC
+# define MODULAR_GC_FN
+#else
+// This takes advantage of internal linkage winning when appearing first.
+// See C99 6.2.2p4.
+# define MODULAR_GC_FN static
+#endif
+
+#if USE_MODULAR_GC
RUBY_SYMBOL_EXPORT_BEGIN
-unsigned int rb_gc_vm_lock(void);
-void rb_gc_vm_unlock(unsigned int lev);
-unsigned int rb_gc_cr_lock(void);
-void rb_gc_cr_unlock(unsigned int lev);
-unsigned int rb_gc_vm_lock_no_barrier(void);
-void rb_gc_vm_unlock_no_barrier(unsigned int lev);
-void rb_gc_vm_barrier(void);
-size_t rb_gc_obj_optimal_size(VALUE obj);
-void rb_gc_mark_children(void *objspace, VALUE obj);
-void rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, vm_table_update_callback_func update_callback, void *data, bool weak_only, enum rb_gc_vm_weak_tables table);
-void rb_gc_update_object_references(void *objspace, VALUE obj);
-void rb_gc_update_vm_references(void *objspace);
-void rb_gc_event_hook(VALUE obj, rb_event_flag_t event);
-void *rb_gc_get_objspace(void);
+#endif
+
+// These functions cannot be defined as static because they are used by other
+// files in Ruby.
size_t rb_size_mul_or_raise(size_t x, size_t y, VALUE exc);
-void rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data);
-void rb_gc_set_pending_interrupt(void);
-void rb_gc_unset_pending_interrupt(void);
-void rb_gc_obj_free_vm_weak_references(VALUE obj);
-bool rb_gc_obj_free(void *objspace, VALUE obj);
-void rb_gc_save_machine_context(void);
-void rb_gc_mark_roots(void *objspace, const char **categoryp);
-void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data);
-bool rb_gc_multi_ractor_p(void);
-void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data);
void rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data);
-void rb_obj_info_dump(VALUE obj);
+const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj);
const char *rb_obj_info(VALUE obj);
-bool rb_gc_shutdown_call_finalizer_p(VALUE obj);
-uint32_t rb_gc_get_shape(VALUE obj);
-void rb_gc_set_shape(VALUE obj, uint32_t shape_id);
-uint32_t rb_gc_rebuild_shape(VALUE obj, size_t heap_id);
size_t rb_obj_memsize_of(VALUE obj);
-void rb_gc_prepare_heap_process_object(VALUE obj);
bool ruby_free_at_exit_p(void);
-bool rb_memerror_reentered(void);
+void rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data);
+void rb_gc_verify_shareable(VALUE);
+
+MODULAR_GC_FN unsigned int rb_gc_vm_lock(const char *file, int line);
+MODULAR_GC_FN void rb_gc_vm_unlock(unsigned int lev, const char *file, int line);
+MODULAR_GC_FN unsigned int rb_gc_cr_lock(const char *file, int line);
+MODULAR_GC_FN void rb_gc_cr_unlock(unsigned int lev, const char *file, int line);
+MODULAR_GC_FN unsigned int rb_gc_vm_lock_no_barrier(const char *file, int line);
+MODULAR_GC_FN void rb_gc_vm_unlock_no_barrier(unsigned int lev, const char *file, int line);
+MODULAR_GC_FN void rb_gc_vm_barrier(void);
+MODULAR_GC_FN size_t rb_gc_obj_optimal_size(VALUE obj);
+MODULAR_GC_FN void rb_gc_mark_children(void *objspace, VALUE obj);
+MODULAR_GC_FN void rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, vm_table_update_callback_func update_callback, void *data, bool weak_only, enum rb_gc_vm_weak_tables table);
+MODULAR_GC_FN void rb_gc_update_object_references(void *objspace, VALUE obj);
+MODULAR_GC_FN void rb_gc_update_vm_references(void *objspace);
+MODULAR_GC_FN void rb_gc_event_hook(VALUE obj, rb_event_flag_t event);
+MODULAR_GC_FN void *rb_gc_get_objspace(void);
+MODULAR_GC_FN void rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void *data), void *data);
+MODULAR_GC_FN void rb_gc_set_pending_interrupt(void);
+MODULAR_GC_FN void rb_gc_unset_pending_interrupt(void);
+MODULAR_GC_FN void rb_gc_obj_free_vm_weak_references(VALUE obj);
+MODULAR_GC_FN bool rb_gc_obj_free(void *objspace, VALUE obj);
+MODULAR_GC_FN void rb_gc_save_machine_context(void);
+MODULAR_GC_FN void rb_gc_mark_roots(void *objspace, const char **categoryp);
+MODULAR_GC_FN void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data);
+MODULAR_GC_FN bool rb_gc_multi_ractor_p(void);
+MODULAR_GC_FN bool rb_gc_shutdown_call_finalizer_p(VALUE obj);
+MODULAR_GC_FN void rb_gc_obj_changed_pool(VALUE obj, size_t heap_id);
+MODULAR_GC_FN void rb_gc_prepare_heap_process_object(VALUE obj);
+MODULAR_GC_FN bool rb_memerror_reentered(void);
+MODULAR_GC_FN bool rb_obj_id_p(VALUE);
+MODULAR_GC_FN void rb_gc_before_updating_jit_code(void);
+MODULAR_GC_FN void rb_gc_after_updating_jit_code(void);
+MODULAR_GC_FN bool rb_gc_obj_shareable_p(VALUE);
+MODULAR_GC_FN void rb_gc_rp(VALUE);
+MODULAR_GC_FN void rb_gc_handle_weak_references(VALUE obj);
+MODULAR_GC_FN bool rb_gc_obj_needs_cleanup_p(VALUE obj);
#if USE_MODULAR_GC
-bool rb_gc_event_hook_required_p(rb_event_flag_t event);
-void *rb_gc_get_ractor_newobj_cache(void);
-void rb_gc_initialize_vm_context(struct rb_gc_vm_context *context);
-void rb_gc_worker_thread_set_vm_context(struct rb_gc_vm_context *context);
-void rb_gc_worker_thread_unset_vm_context(struct rb_gc_vm_context *context);
+MODULAR_GC_FN bool rb_gc_event_hook_required_p(rb_event_flag_t event);
+MODULAR_GC_FN void *rb_gc_get_ractor_newobj_cache(void);
+MODULAR_GC_FN void rb_gc_initialize_vm_context(struct rb_gc_vm_context *context);
+MODULAR_GC_FN void rb_gc_move_obj_during_marking(VALUE from, VALUE to);
+MODULAR_GC_FN void rb_gc_print_backtrace();
#endif
+
+#if USE_MODULAR_GC
RUBY_SYMBOL_EXPORT_END
+#endif
void rb_ractor_finish_marking(void);
@@ -103,7 +156,7 @@ RBIMPL_WARNING_IGNORED(-Wunused-function)
#endif
#ifndef GC_ASSERT
-# define GC_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr)
+# define GC_ASSERT(expr, ...) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__))
#endif
static int
@@ -142,6 +195,14 @@ gc_mark_tbl_no_pin_i(st_data_t key, st_data_t value, st_data_t data)
}
static int
+gc_mark_set_no_pin_i(st_data_t key, st_data_t value, st_data_t data)
+{
+ rb_gc_mark_movable((VALUE)key);
+
+ return ST_CONTINUE;
+}
+
+static int
hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error)
{
if (rb_gc_location((VALUE)key) != (VALUE)key) {
@@ -158,12 +219,14 @@ hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error)
static int
hash_replace_ref(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
{
- if (rb_gc_location((VALUE)*key) != (VALUE)*key) {
- *key = rb_gc_location((VALUE)*key);
+ VALUE new_key = rb_gc_location((VALUE)*key);
+ if (new_key != (VALUE)*key) {
+ *key = new_key;
}
- if (rb_gc_location((VALUE)*value) != (VALUE)*value) {
- *value = rb_gc_location((VALUE)*value);
+ VALUE new_value = rb_gc_location((VALUE)*value);
+ if (new_value != (VALUE)*value) {
+ *value = new_value;
}
return ST_CONTINUE;
diff --git a/gc/gc_impl.h b/gc/gc_impl.h
index 5e8d209432..d9e44cc66d 100644
--- a/gc/gc_impl.h
+++ b/gc/gc_impl.h
@@ -35,14 +35,14 @@ struct rb_gc_object_metadata_entry {
// Bootup
GC_IMPL_FN void *rb_gc_impl_objspace_alloc(void);
GC_IMPL_FN void rb_gc_impl_objspace_init(void *objspace_ptr);
-GC_IMPL_FN void rb_gc_impl_objspace_free(void *objspace_ptr);
GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor);
-GC_IMPL_FN void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache);
GC_IMPL_FN void rb_gc_impl_set_params(void *objspace_ptr);
GC_IMPL_FN void rb_gc_impl_init(void);
GC_IMPL_FN size_t *rb_gc_impl_heap_sizes(void *objspace_ptr);
// Shutdown
GC_IMPL_FN void rb_gc_impl_shutdown_free_objects(void *objspace_ptr);
+GC_IMPL_FN void rb_gc_impl_objspace_free(void *objspace_ptr);
+GC_IMPL_FN void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache);
// GC
GC_IMPL_FN void rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact);
GC_IMPL_FN bool rb_gc_impl_during_gc_p(void *objspace_ptr);
@@ -54,8 +54,9 @@ GC_IMPL_FN void rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag);
GC_IMPL_FN VALUE rb_gc_impl_stress_get(void *objspace_ptr);
GC_IMPL_FN VALUE rb_gc_impl_config_get(void *objspace_ptr);
GC_IMPL_FN void rb_gc_impl_config_set(void *objspace_ptr, VALUE hash);
+GC_IMPL_FN struct rb_gc_vm_context *rb_gc_impl_get_vm_context(void *objspace_ptr);
// Object allocation
-GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size);
+GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size);
GC_IMPL_FN size_t rb_gc_impl_obj_slot_size(VALUE obj);
GC_IMPL_FN size_t rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size);
GC_IMPL_FN bool rb_gc_impl_size_allocatable_p(size_t size);
@@ -72,9 +73,9 @@ GC_IMPL_FN bool rb_gc_impl_size_allocatable_p(size_t size);
* memory just return NULL (with appropriate errno set).
* The caller side takes care of that situation.
*/
-GC_IMPL_FN void *rb_gc_impl_malloc(void *objspace_ptr, size_t size);
-GC_IMPL_FN void *rb_gc_impl_calloc(void *objspace_ptr, size_t size);
-GC_IMPL_FN void *rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size);
+GC_IMPL_FN void *rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed);
+GC_IMPL_FN void *rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed);
+GC_IMPL_FN void *rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed);
GC_IMPL_FN void rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size);
GC_IMPL_FN void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff);
// Marking
@@ -82,9 +83,11 @@ GC_IMPL_FN void rb_gc_impl_mark(void *objspace_ptr, VALUE obj);
GC_IMPL_FN void rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr);
GC_IMPL_FN void rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj);
GC_IMPL_FN void rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj);
-GC_IMPL_FN void rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr);
-GC_IMPL_FN void rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr);
+// Weak references
+GC_IMPL_FN void rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj);
+GC_IMPL_FN bool rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj);
// Compaction
+GC_IMPL_FN void rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj);
GC_IMPL_FN bool rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj);
GC_IMPL_FN VALUE rb_gc_impl_location(void *objspace_ptr, VALUE value);
// Write barriers
@@ -100,9 +103,6 @@ GC_IMPL_FN VALUE rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALU
GC_IMPL_FN void rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj);
GC_IMPL_FN void rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj);
GC_IMPL_FN void rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr);
-// Object ID
-GC_IMPL_FN VALUE rb_gc_impl_object_id(void *objspace_ptr, VALUE obj);
-GC_IMPL_FN VALUE rb_gc_impl_object_id_to_ref(void *objspace_ptr, VALUE object_id);
// Forking
GC_IMPL_FN void rb_gc_impl_before_fork(void *objspace_ptr);
GC_IMPL_FN void rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid);
diff --git a/gc/mmtk/Cargo.lock b/gc/mmtk/Cargo.lock
index f909c3ae33..910048fa80 100644
--- a/gc/mmtk/Cargo.lock
+++ b/gc/mmtk/Cargo.lock
@@ -13,9 +13,9 @@ dependencies = [
[[package]]
name = "anstream"
-version = "0.6.14"
+version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -28,35 +28,36 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.7"
+version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
[[package]]
name = "anstyle-parse"
-version = "0.2.4"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
-version = "1.1.0"
+version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
dependencies = [
"windows-sys",
]
[[package]]
name = "anstyle-wincon"
-version = "3.0.3"
+version = "3.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
+checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa"
dependencies = [
"anstyle",
+ "once_cell_polyfill",
"windows-sys",
]
@@ -87,54 +88,54 @@ checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c"
[[package]]
name = "autocfg"
-version = "1.3.0"
+version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "bitflags"
-version = "2.6.0"
+version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
[[package]]
name = "built"
-version = "0.7.3"
+version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6a6c0b39c38fd754ac338b00a88066436389c0f029da5d37d1e01091d9b7c17"
+checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b"
dependencies = [
"git2",
]
[[package]]
name = "bytemuck"
-version = "1.16.1"
+version = "1.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e"
+checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c"
dependencies = [
"bytemuck_derive",
]
[[package]]
name = "bytemuck_derive"
-version = "1.7.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b"
+checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
name = "cc"
-version = "1.0.100"
+version = "1.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c891175c3fb232128f48de6590095e59198bbeb8620c310be349bfc3afd12c7b"
+checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7"
dependencies = [
"jobserver",
"libc",
- "once_cell",
+ "shlex",
]
[[package]]
@@ -145,15 +146,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "colorchoice"
-version = "1.0.1"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "crossbeam"
@@ -170,18 +171,18 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
-version = "0.5.13"
+version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
-version = "0.8.5"
+version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
@@ -198,41 +199,41 @@ dependencies = [
[[package]]
name = "crossbeam-queue"
-version = "0.3.11"
+version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35"
+checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
-version = "0.8.20"
+version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "delegate"
-version = "0.12.0"
+version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e018fccbeeb50ff26562ece792ed06659b9c2dae79ece77c4456bb10d9bf79b"
+checksum = "b9b6483c2bbed26f97861cf57651d4f2b731964a28cd2257f934a4b452480d21"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
name = "downcast-rs"
-version = "1.2.1"
+version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
+checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf"
[[package]]
name = "either"
-version = "1.12.0"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "enum-map"
@@ -251,14 +252,14 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
name = "env_filter"
-version = "0.1.0"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea"
+checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
dependencies = [
"log",
"regex",
@@ -266,14 +267,14 @@ dependencies = [
[[package]]
name = "env_logger"
-version = "0.11.3"
+version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9"
+checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f"
dependencies = [
"anstream",
"anstyle",
"env_filter",
- "humantime",
+ "jiff",
"log",
]
@@ -287,10 +288,22 @@ dependencies = [
]
[[package]]
+name = "getrandom"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasi",
+]
+
+[[package]]
name = "git2"
-version = "0.18.3"
+version = "0.20.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "232e6a7bfe35766bf715e55a88b39a700596c0ccfd88cd3680b4cdb40d66ef70"
+checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b"
dependencies = [
"bitflags",
"libc",
@@ -312,53 +325,99 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
[[package]]
-name = "humantime"
-version = "2.1.0"
+name = "hermit-abi"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08"
[[package]]
name = "idna"
-version = "0.5.0"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
+checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
dependencies = [
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "279259b0ac81c89d11c290495fdcfa96ea3643b7df311c138b6fe8ca5237f0f8"
+dependencies = [
+ "idna_mapping",
"unicode-bidi",
"unicode-normalization",
]
[[package]]
+name = "idna_mapping"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11c13906586a4b339310541a274dd927aff6fcbb5b8e3af90634c4b31681c792"
+dependencies = [
+ "unicode-joining-type",
+]
+
+[[package]]
name = "is-terminal"
-version = "0.4.12"
+version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
+checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
- "hermit-abi",
+ "hermit-abi 0.5.1",
"libc",
"windows-sys",
]
[[package]]
name = "is_terminal_polyfill"
-version = "1.70.0"
+version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]]
name = "itertools"
-version = "0.12.1"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
dependencies = [
"either",
]
[[package]]
+name = "jiff"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93"
+dependencies = [
+ "jiff-static",
+ "log",
+ "portable-atomic",
+ "portable-atomic-util",
+ "serde",
+]
+
+[[package]]
+name = "jiff-static"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
+[[package]]
name = "jobserver"
-version = "0.1.31"
+version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
+checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
dependencies = [
+ "getrandom",
"libc",
]
@@ -370,15 +429,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
-version = "0.2.155"
+version = "0.2.172"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
[[package]]
name = "libgit2-sys"
-version = "0.16.2+1.7.2"
+version = "0.18.3+1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8"
+checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487"
dependencies = [
"cc",
"libc",
@@ -388,9 +447,9 @@ dependencies = [
[[package]]
name = "libz-sys"
-version = "1.1.18"
+version = "1.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e"
+checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"
dependencies = [
"cc",
"libc",
@@ -410,9 +469,9 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.21"
+version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
+checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "memchr"
@@ -431,20 +490,22 @@ dependencies = [
[[package]]
name = "mmtk"
-version = "0.30.0"
-source = "git+https://github.com/mmtk/mmtk-core.git?rev=051bc7470feef915c445305301e6113f86d3957b#051bc7470feef915c445305301e6113f86d3957b"
+version = "0.31.0"
+source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791"
dependencies = [
"atomic",
"atomic-traits",
"atomic_refcell",
"built",
"bytemuck",
+ "bytemuck_derive",
"cfg-if",
"crossbeam",
"delegate",
"downcast-rs",
"enum-map",
"env_logger",
+ "idna_adapter",
"is-terminal",
"itertools",
"lazy_static",
@@ -456,24 +517,25 @@ dependencies = [
"num_cpus",
"portable-atomic",
"probe",
+ "rayon-core",
"regex",
"rustversion",
"spin",
"static_assertions",
"strum",
"strum_macros",
- "sysinfo 0.30.12",
+ "sysinfo 0.33.1",
]
[[package]]
name = "mmtk-macros"
-version = "0.30.0"
-source = "git+https://github.com/mmtk/mmtk-core.git?rev=051bc7470feef915c445305301e6113f86d3957b#051bc7470feef915c445305301e6113f86d3957b"
+version = "0.31.0"
+source = "git+https://github.com/mmtk/mmtk-core.git?rev=c6317a3f1c262e33fc2e427e4cc999c17bcc4791#c6317a3f1c262e33fc2e427e4cc999c17bcc4791"
dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -487,7 +549,7 @@ dependencies = [
"mmtk",
"once_cell",
"probe",
- "sysinfo 0.32.0",
+ "sysinfo 0.32.1",
]
[[package]]
@@ -514,15 +576,21 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.9",
"libc",
]
[[package]]
name = "once_cell"
-version = "1.19.0"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
[[package]]
name = "percent-encoding"
@@ -532,15 +600,24 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "pkg-config"
-version = "0.3.30"
+version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "portable-atomic"
-version = "1.6.0"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
+checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
+dependencies = [
+ "portable-atomic",
+]
[[package]]
name = "probe"
@@ -574,23 +651,29 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.86"
+version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
-version = "1.0.36"
+version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
dependencies = [
"proc-macro2",
]
[[package]]
+name = "r-efi"
+version = "5.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+
+[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -612,9 +695,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.10.5"
+version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
@@ -624,9 +707,9 @@ dependencies = [
[[package]]
name = "regex-automata"
-version = "0.4.7"
+version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
@@ -635,24 +718,24 @@ dependencies = [
[[package]]
name = "regex-syntax"
-version = "0.8.4"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rustc_version"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustversion"
-version = "1.0.17"
+version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6"
+checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
[[package]]
name = "scopeguard"
@@ -662,9 +745,41 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
-version = "1.0.23"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
+[[package]]
+name = "serde"
+version = "1.0.219"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.219"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "smallvec"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
[[package]]
name = "spin"
@@ -683,21 +798,21 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strum"
-version = "0.26.3"
+version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
+checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32"
[[package]]
name = "strum_macros"
-version = "0.26.4"
+version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
+checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8"
dependencies = [
"heck",
"proc-macro2",
"quote",
"rustversion",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -712,9 +827,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.68"
+version = "2.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9"
+checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
dependencies = [
"proc-macro2",
"quote",
@@ -723,38 +838,37 @@ dependencies = [
[[package]]
name = "sysinfo"
-version = "0.30.12"
+version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae"
+checksum = "4c33cd241af0f2e9e3b5c32163b873b29956890b5342e6745b917ce9d490f4af"
dependencies = [
- "cfg-if",
"core-foundation-sys",
"libc",
+ "memchr",
"ntapi",
- "once_cell",
"rayon",
- "windows 0.52.0",
+ "windows",
]
[[package]]
name = "sysinfo"
-version = "0.32.0"
+version = "0.33.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3b5ae3f4f7d64646c46c4cae4e3f01d1c5d255c7406fdd7c7f999a94e488791"
+checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"
dependencies = [
"core-foundation-sys",
"libc",
"memchr",
"ntapi",
"rayon",
- "windows 0.57.0",
+ "windows",
]
[[package]]
name = "tinyvec"
-version = "1.6.0"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
dependencies = [
"tinyvec_macros",
]
@@ -767,30 +881,36 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "unicode-bidi"
-version = "0.3.15"
+version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
[[package]]
name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
+
+[[package]]
+name = "unicode-joining-type"
+version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "d8d00a78170970967fdb83f9d49b92f959ab2bb829186b113e4f4604ad98e180"
[[package]]
name = "unicode-normalization"
-version = "0.1.23"
+version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
dependencies = [
"tinyvec",
]
[[package]]
name = "url"
-version = "2.5.2"
+version = "2.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
+checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
dependencies = [
"form_urlencoded",
"idna",
@@ -798,6 +918,12 @@ dependencies = [
]
[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
+[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -811,9 +937,18 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "wasi"
+version = "0.14.2+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
+dependencies = [
+ "wit-bindgen-rt",
+]
[[package]]
name = "winapi"
@@ -839,30 +974,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
-dependencies = [
- "windows-core 0.52.0",
- "windows-targets",
-]
-
-[[package]]
-name = "windows"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
dependencies = [
- "windows-core 0.57.0",
- "windows-targets",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
-dependencies = [
+ "windows-core",
"windows-targets",
]
@@ -886,7 +1002,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -897,7 +1013,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.68",
+ "syn 2.0.101",
]
[[package]]
@@ -911,18 +1027,18 @@ dependencies = [
[[package]]
name = "windows-sys"
-version = "0.52.0"
+version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
@@ -936,48 +1052,57 @@ dependencies = [
[[package]]
name = "windows_aarch64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
-version = "0.52.5"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
+dependencies = [
+ "bitflags",
+]
diff --git a/gc/mmtk/Cargo.toml b/gc/mmtk/Cargo.toml
index 66af77fe61..d856122900 100644
--- a/gc/mmtk/Cargo.toml
+++ b/gc/mmtk/Cargo.toml
@@ -21,11 +21,11 @@ probe = "0.5"
sysinfo = "0.32.0"
[dependencies.mmtk]
-features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery", "immix_non_moving"]
+features = ["is_mmtk_object", "object_pinning", "sticky_immix_non_moving_nursery"]
# Uncomment the following lines to use mmtk-core from the official repository.
git = "https://github.com/mmtk/mmtk-core.git"
-rev = "051bc7470feef915c445305301e6113f86d3957b"
+rev = "c6317a3f1c262e33fc2e427e4cc999c17bcc4791"
# Uncomment the following line to use mmtk-core from a local repository.
# path = "../../../mmtk-core"
@@ -35,3 +35,8 @@ default = []
# When moving an object, clear its original copy.
clear_old_copy = []
+
+# Enable extra assertions in release build. For debugging.
+extra_assert = []
+
+[workspace]
diff --git a/gc/mmtk/cbindgen.toml b/gc/mmtk/cbindgen.toml
index c66f829b3d..b99c30efc8 100644
--- a/gc/mmtk/cbindgen.toml
+++ b/gc/mmtk/cbindgen.toml
@@ -20,6 +20,11 @@ typedef void *MMTk_Address;
typedef void *MMTk_ObjectReference;
typedef void *MMTk_NullableObjectReference;
typedef uint32_t MMTk_AllocationSemantics;
+
+typedef struct MMTk_BumpPointer {
+ uintptr_t cursor;
+ uintptr_t limit;
+} MMTk_BumpPointer;
"""
[export]
diff --git a/gc/mmtk/extconf.rb b/gc/mmtk/extconf.rb
index 5f4228972b..c0e788037e 100644
--- a/gc/mmtk/extconf.rb
+++ b/gc/mmtk/extconf.rb
@@ -15,6 +15,10 @@ create_gc_makefile("mmtk") do |makefile|
MMTK_BUILD = debug
LIBMMTK_RUBY = libmmtk_ruby.#$LIBEXT
RUSTSRCS = #{rustsrcs.join(" \\\n\t ")}
+
+ ifeq ($(MMTK_BUILD), debug)
+ CPPFLAGS += -DMMTK_DEBUG
+ endif
MAKEFILE
]
end
diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c
index f52b183fa7..95176b692b 100644
--- a/gc/mmtk/mmtk.c
+++ b/gc/mmtk/mmtk.c
@@ -21,13 +21,10 @@ struct objspace {
bool gc_stress;
size_t gc_count;
+ size_t moving_gc_count;
size_t total_gc_time;
size_t total_allocated_objects;
- st_table *id_to_obj_tbl;
- st_table *obj_to_id_tbl;
- unsigned long long next_object_id;
-
st_table *finalizer_table;
struct MMTk_final_job *finalizer_jobs;
rb_postponed_job_handle_t finalizer_postponed_job;
@@ -36,19 +33,38 @@ struct objspace {
unsigned long live_ractor_cache_count;
pthread_mutex_t mutex;
+ rb_atomic_t mutator_blocking_count;
bool world_stopped;
pthread_cond_t cond_world_stopped;
pthread_cond_t cond_world_started;
size_t start_the_world_count;
+ pthread_mutex_t event_hook_mutex;
+
+ struct {
+ bool gc_thread_crashed;
+ char crash_msg[256];
+ } crash_context;
+
struct rb_gc_vm_context vm_context;
+
+ unsigned int fork_hook_vm_lock_lev;
};
+#define OBJ_FREE_BUF_CAPACITY 128
+
struct MMTk_ractor_cache {
struct ccan_list_node list_node;
MMTk_Mutator *mutator;
bool gc_mutator_p;
+
+ MMTk_BumpPointer *bump_pointer;
+
+ MMTk_ObjectReference obj_free_parallel_buf[OBJ_FREE_BUF_CAPACITY];
+ size_t obj_free_parallel_count;
+ MMTk_ObjectReference obj_free_non_parallel_buf[OBJ_FREE_BUF_CAPACITY];
+ size_t obj_free_non_parallel_count;
};
struct MMTk_final_job {
@@ -63,7 +79,7 @@ struct MMTk_final_job {
void *data;
} dfree;
struct {
- VALUE object_id;
+ /* HACK: we store the object ID on the 0th element of this array. */
VALUE finalizer_array;
} finalize;
} as;
@@ -71,12 +87,22 @@ struct MMTk_final_job {
#ifdef RB_THREAD_LOCAL_SPECIFIER
RB_THREAD_LOCAL_SPECIFIER struct MMTk_GCThreadTLS *rb_mmtk_gc_thread_tls;
+
+RB_THREAD_LOCAL_SPECIFIER VALUE marking_parent_object;
#else
# error We currently need language-supported TLS
#endif
+#ifdef MMTK_DEBUG
+# define MMTK_ASSERT(expr, ...) RUBY_ASSERT_ALWAYS(expr, #expr RBIMPL_VA_OPT_ARGS(__VA_ARGS__))
+#else
+# define MMTK_ASSERT(expr, ...) ((void)0)
+#endif
+
#include <pthread.h>
+static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin);
+
static void
rb_mmtk_init_gc_worker_thread(MMTk_VMWorkerThread gc_thread_tls)
{
@@ -109,7 +135,7 @@ rb_mmtk_stop_the_world(void)
}
static void
-rb_mmtk_resume_mutators(void)
+rb_mmtk_resume_mutators(bool current_gc_may_move)
{
struct objspace *objspace = rb_gc_get_objspace();
@@ -120,6 +146,7 @@ rb_mmtk_resume_mutators(void)
objspace->world_stopped = false;
objspace->gc_count++;
+ if (current_gc_may_move) objspace->moving_gc_count++;
pthread_cond_broadcast(&objspace->cond_world_started);
if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) {
@@ -127,13 +154,17 @@ rb_mmtk_resume_mutators(void)
}
}
+static void mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache);
+
static void
rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
{
struct objspace *objspace = rb_gc_get_objspace();
size_t starting_gc_count = objspace->gc_count;
- int lock_lev = rb_gc_vm_lock();
+ RUBY_ATOMIC_INC(objspace->mutator_blocking_count);
+ int lock_lev = RB_GC_VM_LOCK();
+ RUBY_ATOMIC_DEC(objspace->mutator_blocking_count);
int err;
if ((err = pthread_mutex_lock(&objspace->mutex)) != 0) {
rb_bug("ERROR: cannot lock objspace->mutex: %s", strerror(err));
@@ -155,6 +186,11 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
rb_gc_vm_barrier();
+ struct MMTk_ractor_cache *rc;
+ ccan_list_for_each(&objspace->ractor_caches, rc, list_node) {
+ mmtk_flush_obj_free_buffer(rc);
+ }
+
objspace->world_stopped = true;
pthread_cond_broadcast(&objspace->cond_world_stopped);
@@ -164,6 +200,10 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
pthread_cond_wait(&objspace->cond_world_started, &objspace->mutex);
}
+ if (RB_UNLIKELY(objspace->crash_context.gc_thread_crashed)) {
+ rb_bug("%s", objspace->crash_context.crash_msg);
+ }
+
if (objspace->measure_gc_time) {
struct timespec gc_end_time;
clock_gettime(CLOCK_MONOTONIC, &gc_end_time);
@@ -177,7 +217,19 @@ rb_mmtk_block_for_gc(MMTk_VMMutatorThread mutator)
if ((err = pthread_mutex_unlock(&objspace->mutex)) != 0) {
rb_bug("ERROR: cannot release objspace->mutex: %s", strerror(err));
}
- rb_gc_vm_unlock(lock_lev);
+ RB_GC_VM_UNLOCK(lock_lev);
+}
+
+static void
+rb_mmtk_before_updating_jit_code(void)
+{
+ rb_gc_before_updating_jit_code();
+}
+
+static void
+rb_mmtk_after_updating_jit_code(void)
+{
+ rb_gc_after_updating_jit_code();
}
static size_t
@@ -191,9 +243,7 @@ static void
rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), void *data)
{
struct objspace *objspace = rb_gc_get_objspace();
-
struct MMTk_ractor_cache *ractor_cache;
- RUBY_ASSERT(ractor_cache != NULL);
ccan_list_for_each(&objspace->ractor_caches, ractor_cache, list_node) {
visit_mutator(ractor_cache->mutator, data);
@@ -203,7 +253,9 @@ rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), v
static void
rb_mmtk_scan_gc_roots(void)
{
- // rb_gc_mark_roots(rb_gc_get_objspace(), NULL);
+ struct objspace *objspace = rb_gc_get_objspace();
+
+ rb_gc_mark_roots(objspace, NULL);
}
static int
@@ -223,15 +275,12 @@ rb_mmtk_scan_objspace(void)
st_foreach(objspace->finalizer_table, pin_value, (st_data_t)objspace);
}
- st_foreach(objspace->obj_to_id_tbl, gc_mark_tbl_no_pin_i, (st_data_t)objspace);
-
struct MMTk_final_job *job = objspace->finalizer_jobs;
while (job != NULL) {
switch (job->kind) {
case MMTK_FINAL_JOB_DFREE:
break;
case MMTK_FINAL_JOB_FINALIZE:
- rb_gc_impl_mark(objspace, job->as.finalize.object_id);
rb_gc_impl_mark(objspace, job->as.finalize.finalizer_array);
break;
default:
@@ -243,27 +292,45 @@ rb_mmtk_scan_objspace(void)
}
static void
-rb_mmtk_scan_roots_in_mutator_thread(MMTk_VMMutatorThread mutator, MMTk_VMWorkerThread worker)
+rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to)
+{
+ rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to);
+}
+
+static void
+rb_mmtk_update_object_references(MMTk_ObjectReference mmtk_object)
{
- if (mutator->gc_mutator_p) {
- struct objspace *objspace = rb_gc_get_objspace();
+ VALUE object = (VALUE)mmtk_object;
- rb_gc_worker_thread_set_vm_context(&objspace->vm_context);
- rb_gc_mark_roots(objspace, NULL);
- rb_gc_worker_thread_unset_vm_context(&objspace->vm_context);
+ if (!RB_FL_TEST(object, RUBY_FL_WEAK_REFERENCE)) {
+ marking_parent_object = object;
+ rb_gc_update_object_references(rb_gc_get_objspace(), object);
+ marking_parent_object = 0;
}
}
static void
-rb_mmtk_scan_object_ruby_style(MMTk_ObjectReference object)
+rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object)
{
+ marking_parent_object = (VALUE)object;
rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object);
+ marking_parent_object = 0;
}
static void
-rb_mmtk_call_gc_mark_children(MMTk_ObjectReference object)
+rb_mmtk_handle_weak_references(MMTk_ObjectReference mmtk_object, bool moving)
{
- rb_gc_mark_children(rb_gc_get_objspace(), (VALUE)object);
+ VALUE object = (VALUE)mmtk_object;
+
+ marking_parent_object = object;
+
+ rb_gc_handle_weak_references(object);
+
+ if (moving) {
+ rb_gc_update_object_references(rb_gc_get_objspace(), object);
+ }
+
+ marking_parent_object = 0;
}
static void
@@ -273,12 +340,16 @@ rb_mmtk_call_obj_free(MMTk_ObjectReference object)
struct objspace *objspace = rb_gc_get_objspace();
if (RB_UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_FREEOBJ))) {
- rb_gc_worker_thread_set_vm_context(&objspace->vm_context);
+ pthread_mutex_lock(&objspace->event_hook_mutex);
rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ);
- rb_gc_worker_thread_unset_vm_context(&objspace->vm_context);
+ pthread_mutex_unlock(&objspace->event_hook_mutex);
}
rb_gc_obj_free(objspace, obj);
+
+#ifdef MMTK_DEBUG
+ memset((void *)obj, 0, rb_gc_impl_obj_slot_size(obj));
+#endif
}
static size_t
@@ -290,31 +361,34 @@ rb_mmtk_vm_live_bytes(void)
static void
make_final_job(struct objspace *objspace, VALUE obj, VALUE table)
{
- RUBY_ASSERT(RB_FL_TEST(obj, RUBY_FL_FINALIZE));
- RUBY_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)table));
- RUBY_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY);
-
- RB_FL_UNSET(obj, RUBY_FL_FINALIZE);
+ MMTK_ASSERT(RB_BUILTIN_TYPE(table) == T_ARRAY);
struct MMTk_final_job *job = xmalloc(sizeof(struct MMTk_final_job));
job->next = objspace->finalizer_jobs;
job->kind = MMTK_FINAL_JOB_FINALIZE;
- job->as.finalize.object_id = rb_obj_id((VALUE)obj);
job->as.finalize.finalizer_array = table;
objspace->finalizer_jobs = job;
}
static int
-rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data)
+rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data, int error)
{
- RUBY_ASSERT(RB_FL_TEST(key, RUBY_FL_FINALIZE));
- RUBY_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value));
- RUBY_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY);
+ MMTK_ASSERT(mmtk_is_reachable((MMTk_ObjectReference)value));
+ MMTK_ASSERT(RB_BUILTIN_TYPE(value) == T_ARRAY);
struct objspace *objspace = (struct objspace *)data;
- if (!mmtk_is_reachable((MMTk_ObjectReference)key)) {
+ if (mmtk_is_reachable((MMTk_ObjectReference)key)) {
+ VALUE new_key_location = rb_mmtk_call_object_closure((VALUE)key, false);
+
+ MMTK_ASSERT(RB_FL_TEST(new_key_location, RUBY_FL_FINALIZE));
+
+ if (new_key_location != key) {
+ return ST_REPLACE;
+ }
+ }
+ else {
make_final_job(objspace, (VALUE)key, (VALUE)value);
rb_postponed_job_trigger(objspace->finalizer_postponed_job);
@@ -325,70 +399,119 @@ rb_mmtk_update_finalizer_table_i(st_data_t key, st_data_t value, st_data_t data)
return ST_CONTINUE;
}
+static int
+rb_mmtk_update_finalizer_table_replace_i(st_data_t *key, st_data_t *value, st_data_t data, int existing)
+{
+ *key = rb_mmtk_call_object_closure((VALUE)*key, false);
+
+ return ST_CONTINUE;
+}
+
static void
rb_mmtk_update_finalizer_table(void)
{
struct objspace *objspace = rb_gc_get_objspace();
- // TODO: replace with st_foreach_with_replace when GC is moving
- st_foreach(objspace->finalizer_table, rb_mmtk_update_finalizer_table_i, (st_data_t)objspace);
+ st_foreach_with_replace(
+ objspace->finalizer_table,
+ rb_mmtk_update_finalizer_table_i,
+ rb_mmtk_update_finalizer_table_replace_i,
+ (st_data_t)objspace
+ );
}
static int
-rb_mmtk_update_table_i(VALUE val, void *data)
+rb_mmtk_global_tables_count(void)
+{
+ return RB_GC_VM_WEAK_TABLE_COUNT;
+}
+
+static inline VALUE rb_mmtk_call_object_closure(VALUE obj, bool pin);
+
+static int
+rb_mmtk_update_global_tables_i(VALUE val, void *data)
{
if (!mmtk_is_reachable((MMTk_ObjectReference)val)) {
return ST_DELETE;
}
+ // TODO: check only if in moving GC
+ if (rb_mmtk_call_object_closure(val, false) != val) {
+ return ST_REPLACE;
+ }
+
return ST_CONTINUE;
}
static int
-rb_mmtk_update_obj_id_tables_obj_to_id_i(st_data_t key, st_data_t val, st_data_t data)
+rb_mmtk_update_global_tables_replace_i(VALUE *ptr, void *data)
{
- RUBY_ASSERT(RB_FL_TEST(key, FL_SEEN_OBJ_ID));
-
- if (!mmtk_is_reachable((MMTk_ObjectReference)key)) {
- return ST_DELETE;
- }
+ // TODO: cache the new location so we don't call rb_mmtk_call_object_closure twice
+ *ptr = rb_mmtk_call_object_closure(*ptr, false);
return ST_CONTINUE;
}
-static int
-rb_mmtk_update_obj_id_tables_id_to_obj_i(st_data_t key, st_data_t val, st_data_t data)
+static void
+rb_mmtk_update_global_tables(int table, bool moving)
{
- RUBY_ASSERT(RB_FL_TEST(val, FL_SEEN_OBJ_ID));
+ MMTK_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT);
- if (!mmtk_is_reachable((MMTk_ObjectReference)val)) {
- return ST_DELETE;
- }
+ rb_gc_vm_weak_table_foreach(
+ rb_mmtk_update_global_tables_i,
+ rb_mmtk_update_global_tables_replace_i,
+ NULL,
+ !moving,
+ (enum rb_gc_vm_weak_tables)table
+ );
+}
- return ST_CONTINUE;
+static bool
+rb_mmtk_special_const_p(MMTk_ObjectReference object)
+{
+ VALUE obj = (VALUE)object;
+
+ return RB_SPECIAL_CONST_P(obj);
}
+RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 1, 2)
+RBIMPL_ATTR_NORETURN()
static void
-rb_mmtk_update_obj_id_tables(void)
+rb_mmtk_gc_thread_bug(const char *msg, ...)
{
struct objspace *objspace = rb_gc_get_objspace();
- st_foreach(objspace->obj_to_id_tbl, rb_mmtk_update_obj_id_tables_obj_to_id_i, 0);
- st_foreach(objspace->id_to_obj_tbl, rb_mmtk_update_obj_id_tables_id_to_obj_i, 0);
+ objspace->crash_context.gc_thread_crashed = true;
+
+ va_list args;
+ va_start(args, msg);
+ vsnprintf(objspace->crash_context.crash_msg, sizeof(objspace->crash_context.crash_msg), msg, args);
+ va_end(args);
+
+ fprintf(stderr, "-- GC thread backtrace "
+ "-------------------------------------------\n");
+ rb_gc_print_backtrace();
+ fprintf(stderr, "\n");
+
+ rb_mmtk_resume_mutators(false);
+
+ sleep(5);
+
+ rb_bug("rb_mmtk_gc_thread_bug");
}
-static int
-rb_mmtk_global_tables_count(void)
+RBIMPL_ATTR_NORETURN()
+static void
+rb_mmtk_gc_thread_panic_handler(void)
{
- return RB_GC_VM_WEAK_TABLE_COUNT;
+ rb_mmtk_gc_thread_bug("MMTk GC thread panicked");
}
+RBIMPL_ATTR_NORETURN()
static void
-rb_mmtk_update_global_tables(int table)
+rb_mmtk_mutator_thread_panic_handler(void)
{
- RUBY_ASSERT(table < RB_GC_VM_WEAK_TABLE_COUNT);
-
- rb_gc_vm_weak_table_foreach(rb_mmtk_update_table_i, NULL, NULL, true, (enum rb_gc_vm_weak_tables)table);
+ rb_bug("Ruby mutator thread panicked");
}
// Bootup
@@ -398,19 +521,24 @@ MMTk_RubyUpcalls ruby_upcalls = {
rb_mmtk_stop_the_world,
rb_mmtk_resume_mutators,
rb_mmtk_block_for_gc,
+ rb_mmtk_before_updating_jit_code,
+ rb_mmtk_after_updating_jit_code,
rb_mmtk_number_of_mutators,
rb_mmtk_get_mutators,
rb_mmtk_scan_gc_roots,
rb_mmtk_scan_objspace,
- rb_mmtk_scan_roots_in_mutator_thread,
- rb_mmtk_scan_object_ruby_style,
+ rb_mmtk_move_obj_during_marking,
+ rb_mmtk_update_object_references,
rb_mmtk_call_gc_mark_children,
+ rb_mmtk_handle_weak_references,
rb_mmtk_call_obj_free,
rb_mmtk_vm_live_bytes,
rb_mmtk_update_global_tables,
rb_mmtk_global_tables_count,
rb_mmtk_update_finalizer_table,
- rb_mmtk_update_obj_id_tables,
+ rb_mmtk_special_const_p,
+ rb_mmtk_mutator_thread_panic_handler,
+ rb_mmtk_gc_thread_panic_handler,
};
// Use max 80% of the available memory by default for MMTk
@@ -434,12 +562,14 @@ void *
rb_gc_impl_objspace_alloc(void)
{
MMTk_Builder *builder = rb_mmtk_builder_init();
- mmtk_init_binding(builder, NULL, &ruby_upcalls, (MMTk_ObjectReference)Qundef);
+ MMTk_RubyBindingOptions binding_options = {
+ .suffix_size = RB_GC_OBJ_SUFFIX_SIZE,
+ };
+ mmtk_init_binding(builder, &binding_options, &ruby_upcalls);
return calloc(1, sizeof(struct objspace));
}
-static void objspace_obj_id_init(struct objspace *objspace);
static void gc_run_finalizers(void *data);
void
@@ -449,8 +579,6 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
objspace->measure_gc_time = true;
- objspace_obj_id_init(objspace);
-
objspace->finalizer_table = st_init_numtable();
objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace);
@@ -459,6 +587,8 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
objspace->mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
objspace->cond_world_stopped = (pthread_cond_t)PTHREAD_COND_INITIALIZER;
objspace->cond_world_started = (pthread_cond_t)PTHREAD_COND_INITIALIZER;
+
+ objspace->event_hook_mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
}
void
@@ -476,10 +606,11 @@ rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
}
objspace->live_ractor_cache_count++;
- struct MMTk_ractor_cache *cache = malloc(sizeof(struct MMTk_ractor_cache));
+ struct MMTk_ractor_cache *cache = calloc(1, sizeof(struct MMTk_ractor_cache));
ccan_list_add(&objspace->ractor_caches, &cache->list_node);
cache->mutator = mmtk_bind_mutator(cache);
+ cache->bump_pointer = mmtk_get_bump_pointer_allocator(cache->mutator);
return cache;
}
@@ -492,7 +623,15 @@ rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache_ptr)
ccan_list_del(&cache->list_node);
- RUBY_ASSERT(objspace->live_ractor_cache_count > 1);
+ mmtk_flush_obj_free_buffer(cache);
+
+ if (ruby_free_at_exit_p()) {
+ MMTK_ASSERT(objspace->live_ractor_cache_count > 0);
+ }
+ else {
+ MMTK_ASSERT(objspace->live_ractor_cache_count > 1);
+ }
+
objspace->live_ractor_cache_count--;
mmtk_destroy_mutator(cache->mutator);
@@ -502,15 +641,31 @@ void rb_gc_impl_set_params(void *objspace_ptr) { }
static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; }
+#if SIZEOF_VALUE >= 8
+#define MMTK_HEAP_COUNT 12
+#define MMTK_MAX_OBJ_SIZE 1024
+static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
+ 32, 40, 64, 80, 96, 128, 160, 256, 512, 640, 768, MMTK_MAX_OBJ_SIZE, 0
+};
+#else
+#define MMTK_HEAP_COUNT 5
+#define MMTK_MAX_OBJ_SIZE 512
+static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
+ 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0
+};
+#endif
+
void
rb_gc_impl_init(void)
{
VALUE gc_constants = rb_hash_new();
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(sizeof(VALUE) * 5));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0));
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(640));
- // Pretend we have 5 size pools
- rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(5));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(MMTK_HEAP_COUNT));
+ // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0));
OBJ_FREEZE(gc_constants);
rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
@@ -524,10 +679,6 @@ rb_gc_impl_init(void)
rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
}
-static size_t heap_sizes[6] = {
- 40, 80, 160, 320, 640, 0
-};
-
size_t *
rb_gc_impl_heap_sizes(void *objspace_ptr)
{
@@ -568,8 +719,8 @@ rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool i
bool
rb_gc_impl_during_gc_p(void *objspace_ptr)
{
- // TODO
- return false;
+ struct objspace *objspace = objspace_ptr;
+ return objspace->world_stopped;
}
static void
@@ -639,17 +790,106 @@ rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
// TODO
}
+struct rb_gc_vm_context *
+rb_gc_impl_get_vm_context(void *objspace_ptr)
+{
+ struct objspace *objspace = objspace_ptr;
+
+ return &objspace->vm_context;
+}
+
// Object allocation
+static VALUE
+rb_mmtk_alloc_fast_path(struct objspace *objspace, struct MMTk_ractor_cache *ractor_cache, size_t size, size_t align)
+{
+ MMTk_BumpPointer *bump_pointer = ractor_cache->bump_pointer;
+ if (bump_pointer == NULL) return 0;
+
+ uintptr_t cursor = bump_pointer->cursor;
+
+ // Ensure cursor is aligned
+ size_t mask = align - 1;
+ cursor = (cursor + mask) & ~mask;
+
+ cursor += size;
+
+ if (cursor > bump_pointer->limit) {
+ return 0;
+ }
+ else {
+ VALUE obj = cursor - size;
+ bump_pointer->cursor = cursor;
+ return obj;
+ }
+}
+
+static bool
+obj_can_parallel_free_p(VALUE obj)
+{
+ switch (RB_BUILTIN_TYPE(obj)) {
+ case T_ARRAY:
+ case T_BIGNUM:
+ case T_COMPLEX:
+ case T_FLOAT:
+ case T_HASH:
+ case T_OBJECT:
+ case T_RATIONAL:
+ case T_REGEXP:
+ case T_STRING:
+ case T_STRUCT:
+ case T_SYMBOL:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static void
+mmtk_flush_obj_free_buffer(struct MMTk_ractor_cache *cache)
+{
+ if (cache->obj_free_parallel_count > 0) {
+ mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf,
+ cache->obj_free_parallel_count, true);
+ cache->obj_free_parallel_count = 0;
+ }
+ if (cache->obj_free_non_parallel_count > 0) {
+ mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf,
+ cache->obj_free_non_parallel_count, false);
+ cache->obj_free_non_parallel_count = 0;
+ }
+}
+
+static inline void
+mmtk_buffer_obj_free_candidate(struct MMTk_ractor_cache *cache, VALUE obj)
+{
+ if (obj_can_parallel_free_p(obj)) {
+ cache->obj_free_parallel_buf[cache->obj_free_parallel_count++] = (MMTk_ObjectReference)obj;
+ if (cache->obj_free_parallel_count >= OBJ_FREE_BUF_CAPACITY) {
+ mmtk_add_obj_free_candidates(cache->obj_free_parallel_buf,
+ cache->obj_free_parallel_count, true);
+ cache->obj_free_parallel_count = 0;
+ }
+ }
+ else {
+ cache->obj_free_non_parallel_buf[cache->obj_free_non_parallel_count++] = (MMTk_ObjectReference)obj;
+ if (cache->obj_free_non_parallel_count >= OBJ_FREE_BUF_CAPACITY) {
+ mmtk_add_obj_free_candidates(cache->obj_free_non_parallel_buf,
+ cache->obj_free_non_parallel_count, false);
+ cache->obj_free_non_parallel_count = 0;
+ }
+ }
+}
+
VALUE
-rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size)
+rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
{
#define MMTK_ALLOCATION_SEMANTICS_DEFAULT 0
struct objspace *objspace = objspace_ptr;
struct MMTk_ractor_cache *ractor_cache = cache_ptr;
- if (alloc_size > 640) rb_bug("too big");
- for (int i = 0; i < 5; i++) {
+ if (alloc_size > MMTK_MAX_OBJ_SIZE) rb_bug("too big");
+ for (int i = 0; i < MMTK_HEAP_COUNT; i++) {
if (alloc_size == heap_sizes[i]) break;
if (alloc_size < heap_sizes[i]) {
alloc_size = heap_sizes[i];
@@ -661,19 +901,24 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
mmtk_handle_user_collection_request(ractor_cache, false, false);
}
- VALUE *alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size + 8, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+ // Layout: [hidden size header (sizeof(VALUE))][payload (alloc_size)][suffix (RB_GC_OBJ_SUFFIX_SIZE)]
+ alloc_size += sizeof(VALUE) + RB_GC_OBJ_SUFFIX_SIZE;
+
+ VALUE *alloc_obj = (VALUE *)rb_mmtk_alloc_fast_path(objspace, ractor_cache, alloc_size, MMTk_MIN_OBJ_ALIGN);
+ if (!alloc_obj) {
+ alloc_obj = mmtk_alloc(ractor_cache->mutator, alloc_size, MMTk_MIN_OBJ_ALIGN, 0, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+ }
+
alloc_obj++;
- alloc_obj[-1] = alloc_size;
+ alloc_obj[-1] = alloc_size - sizeof(VALUE) - RB_GC_OBJ_SUFFIX_SIZE;
alloc_obj[0] = flags;
alloc_obj[1] = klass;
- if (alloc_size > 16) alloc_obj[2] = v1;
- if (alloc_size > 24) alloc_obj[3] = v2;
- if (alloc_size > 32) alloc_obj[4] = v3;
- mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size + 8, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
+ // TODO: implement fast path for mmtk_post_alloc
+ mmtk_post_alloc(ractor_cache->mutator, (void*)alloc_obj, alloc_size, MMTK_ALLOCATION_SEMANTICS_DEFAULT);
// TODO: only add when object needs obj_free to be called
- mmtk_add_obj_free_candidate(alloc_obj);
+ mmtk_buffer_obj_free_candidate(ractor_cache, (VALUE)alloc_obj);
objspace->total_allocated_objects++;
@@ -689,7 +934,7 @@ rb_gc_impl_obj_slot_size(VALUE obj)
size_t
rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
{
- for (int i = 0; i < 5; i++) {
+ for (int i = 0; i < MMTK_HEAP_COUNT; i++) {
if (size == heap_sizes[i]) return i;
if (size < heap_sizes[i]) return i;
}
@@ -700,26 +945,26 @@ rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
bool
rb_gc_impl_size_allocatable_p(size_t size)
{
- return size <= 640;
+ return size <= MMTK_MAX_OBJ_SIZE;
}
// Malloc
void *
-rb_gc_impl_malloc(void *objspace_ptr, size_t size)
+rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
// TODO: don't use system malloc
return malloc(size);
}
void *
-rb_gc_impl_calloc(void *objspace_ptr, size_t size)
+rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
{
// TODO: don't use system calloc
return calloc(1, size);
}
void *
-rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size)
+rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
{
// TODO: don't use system realloc
return realloc(ptr, new_size);
@@ -735,15 +980,34 @@ rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
void rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff) { }
// Marking
+static inline VALUE
+rb_mmtk_call_object_closure(VALUE obj, bool pin)
+{
+ if (RB_UNLIKELY(RB_BUILTIN_TYPE(obj) == T_NONE)) {
+ enum { info_size = 256 };
+ char obj_info_buf[info_size];
+ rb_raw_obj_info(obj_info_buf, info_size, obj);
+
+ char parent_obj_info_buf[info_size];
+ rb_raw_obj_info(parent_obj_info_buf, info_size, marking_parent_object);
+
+ rb_mmtk_gc_thread_bug("try to mark T_NONE object (obj: %s, parent: %s)", obj_info_buf, parent_obj_info_buf);
+ }
+
+ return (VALUE)rb_mmtk_gc_thread_tls->object_closure.c_function(
+ rb_mmtk_gc_thread_tls->object_closure.rust_closure,
+ rb_mmtk_gc_thread_tls->gc_context,
+ (MMTk_ObjectReference)obj,
+ pin
+ );
+}
+
void
rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
{
if (RB_SPECIAL_CONST_P(obj)) return;
- rb_mmtk_gc_thread_tls->object_closure.c_function(rb_mmtk_gc_thread_tls->object_closure.rust_closure,
- rb_mmtk_gc_thread_tls->gc_context,
- (MMTk_ObjectReference)obj,
- false);
+ rb_mmtk_call_object_closure(obj, false);
}
void
@@ -751,8 +1015,10 @@ rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
{
if (RB_SPECIAL_CONST_P(*ptr)) return;
- // TODO: make it movable
- rb_gc_impl_mark(objspace_ptr, *ptr);
+ VALUE new_obj = rb_mmtk_call_object_closure(*ptr, false);
+ if (new_obj != *ptr) {
+ *ptr = new_obj;
+ }
}
void
@@ -760,8 +1026,7 @@ rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
{
if (RB_SPECIAL_CONST_P(obj)) return;
- // TODO: also pin
- rb_gc_impl_mark(objspace_ptr, obj);
+ rb_mmtk_call_object_closure(obj, true);
}
void
@@ -773,28 +1038,35 @@ rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
}
void
-rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr)
+rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
{
- mmtk_mark_weak((MMTk_ObjectReference *)ptr);
+ RB_FL_SET(obj, RUBY_FL_WEAK_REFERENCE);
+ mmtk_declare_weak_references((MMTk_ObjectReference)obj);
}
-void
-rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr)
+bool
+rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
{
- mmtk_remove_weak((MMTk_ObjectReference *)ptr);
+ return mmtk_weak_references_alive_p((MMTk_ObjectReference)obj);
}
// Compaction
+void
+rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
+{
+ mmtk_register_pinning_obj((MMTk_ObjectReference)obj);
+}
+
bool
rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
{
- rb_bug("unimplemented");
+ return rb_mmtk_call_object_closure(obj, false) != obj;
}
VALUE
-rb_gc_impl_location(void *objspace_ptr, VALUE value)
+rb_gc_impl_location(void *objspace_ptr, VALUE obj)
{
- rb_bug("unimplemented");
+ return rb_mmtk_call_object_closure(obj, false);
}
// Write barriers
@@ -803,6 +1075,23 @@ rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
{
struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache();
+ if (SPECIAL_CONST_P(b)) return;
+
+#ifdef MMTK_DEBUG
+ if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)a)) {
+ char buff[256];
+ rb_bug("a: %s is not an object", rb_raw_obj_info(buff, 256, a));
+ }
+
+ if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)b)) {
+ char buff[256];
+ rb_bug("b: %s is not an object", rb_raw_obj_info(buff, 256, b));
+ }
+#endif
+
+ MMTK_ASSERT(BUILTIN_TYPE(a) != T_NONE);
+ MMTK_ASSERT(BUILTIN_TYPE(b) != T_NONE);
+
mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)a);
}
@@ -907,7 +1196,7 @@ gc_run_finalizers_get_final(long i, void *data)
{
VALUE table = (VALUE)data;
- return RARRAY_AREF(table, i);
+ return RARRAY_AREF(table, i + 1);
}
static void
@@ -926,17 +1215,15 @@ gc_run_finalizers(void *data)
job->as.dfree.func(job->as.dfree.data);
break;
case MMTK_FINAL_JOB_FINALIZE: {
- VALUE object_id = job->as.finalize.object_id;
VALUE finalizer_array = job->as.finalize.finalizer_array;
rb_gc_run_obj_finalizer(
- job->as.finalize.object_id,
- RARRAY_LEN(finalizer_array),
+ RARRAY_AREF(finalizer_array, 0),
+ RARRAY_LEN(finalizer_array) - 1,
gc_run_finalizers_get_final,
(void *)finalizer_array
);
- RB_GC_GUARD(object_id);
RB_GC_GUARD(finalizer_array);
break;
}
@@ -980,6 +1267,8 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
RBASIC(obj)->flags |= FL_FINALIZE;
+ int lev = RB_GC_VM_LOCK();
+
if (st_lookup(objspace->finalizer_table, obj, &data)) {
table = (VALUE)data;
@@ -991,6 +1280,7 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
for (i = 0; i < len; i++) {
VALUE recv = RARRAY_AREF(table, i);
if (rb_equal(recv, block)) {
+ RB_GC_VM_UNLOCK(lev);
return recv;
}
}
@@ -999,11 +1289,13 @@ rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
rb_ary_push(table, block);
}
else {
- table = rb_ary_new3(1, block);
+ table = rb_ary_new3(2, rb_obj_id(obj), block);
rb_obj_hide(table);
st_add_direct(objspace->finalizer_table, obj, table);
}
+ RB_GC_VM_UNLOCK(lev);
+
return block;
}
@@ -1013,7 +1305,11 @@ rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
struct objspace *objspace = objspace_ptr;
st_data_t data = obj;
+
+ int lev = RB_GC_VM_LOCK();
st_delete(objspace->finalizer_table, &data, 0);
+ RB_GC_VM_UNLOCK(lev);
+
FL_UNSET(obj, FL_FINALIZE);
}
@@ -1026,14 +1322,17 @@ rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
if (!FL_TEST(obj, FL_FINALIZE)) return;
+ int lev = RB_GC_VM_LOCK();
if (RB_LIKELY(st_lookup(objspace->finalizer_table, obj, &data))) {
- table = (VALUE)data;
+ table = rb_ary_dup((VALUE)data);
+ RARRAY_ASET(table, 0, rb_obj_id(dest));
st_insert(objspace->finalizer_table, dest, table);
FL_SET(dest, FL_FINALIZE);
}
else {
rb_bug("rb_gc_copy_finalizer: FL_FINALIZE set but not found in finalizer_table: %s", rb_obj_info(obj));
}
+ RB_GC_VM_UNLOCK(lev);
}
static int
@@ -1057,112 +1356,67 @@ rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
gc_run_finalizers(objspace);
}
- struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates();
- for (size_t i = 0; i < registered_candidates.len; i++) {
- VALUE obj = (VALUE)registered_candidates.ptr[i];
-
- if (rb_gc_shutdown_call_finalizer_p(obj)) {
- rb_gc_obj_free(objspace_ptr, obj);
- RBASIC(obj)->flags = 0;
+ unsigned int lev = RB_GC_VM_LOCK();
+ {
+ struct MMTk_ractor_cache *rc;
+ ccan_list_for_each(&objspace->ractor_caches, rc, list_node) {
+ mmtk_flush_obj_free_buffer(rc);
}
- }
- mmtk_free_raw_vec_of_obj_ref(registered_candidates);
- gc_run_finalizers(objspace);
-}
+ struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates();
+ for (size_t i = 0; i < registered_candidates.len; i++) {
+ VALUE obj = (VALUE)registered_candidates.ptr[i];
-// Object ID
-static int
-object_id_cmp(st_data_t x, st_data_t y)
-{
- if (RB_TYPE_P(x, T_BIGNUM)) {
- return !rb_big_eql(x, y);
- }
- else {
- return x != y;
+ if (rb_gc_shutdown_call_finalizer_p(obj)) {
+ rb_gc_obj_free(objspace_ptr, obj);
+ RBASIC(obj)->flags = 0;
+ }
+ }
+ mmtk_free_raw_vec_of_obj_ref(registered_candidates);
}
-}
-
-static st_index_t
-object_id_hash(st_data_t n)
-{
- return FIX2LONG(rb_hash((VALUE)n));
-}
-
-#define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1)
-#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT)
-
-static const struct st_hash_type object_id_hash_type = {
- object_id_cmp,
- object_id_hash,
-};
+ RB_GC_VM_UNLOCK(lev);
-static void
-objspace_obj_id_init(struct objspace *objspace)
-{
- objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
- objspace->obj_to_id_tbl = st_init_numtable();
- objspace->next_object_id = OBJ_ID_INITIAL;
+ gc_run_finalizers(objspace);
}
-VALUE
-rb_gc_impl_object_id(void *objspace_ptr, VALUE obj)
-{
- struct objspace *objspace = objspace_ptr;
-
- unsigned int lev = rb_gc_vm_lock();
-
- VALUE id;
- if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &id)) {
- RUBY_ASSERT(FL_TEST(obj, FL_SEEN_OBJ_ID));
- }
- else {
- RUBY_ASSERT(!FL_TEST(obj, FL_SEEN_OBJ_ID));
-
- id = ULL2NUM(objspace->next_object_id);
- objspace->next_object_id += OBJ_ID_INCREMENT;
-
- st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
- st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
- FL_SET(obj, FL_SEEN_OBJ_ID);
- }
-
- rb_gc_vm_unlock(lev);
-
- return id;
-}
+// Forking
-VALUE
-rb_gc_impl_object_id_to_ref(void *objspace_ptr, VALUE object_id)
+void
+rb_gc_impl_before_fork(void *objspace_ptr)
{
struct objspace *objspace = objspace_ptr;
- VALUE obj;
- if (st_lookup(objspace->id_to_obj_tbl, object_id, &obj) &&
- !rb_gc_impl_garbage_object_p(objspace, obj)) {
- return obj;
+ retry:
+ objspace->fork_hook_vm_lock_lev = RB_GC_VM_LOCK();
+ rb_gc_vm_barrier();
+
+ /* At this point, we know that all the Ractors are paused because of the
+ * rb_gc_vm_barrier above. Since rb_mmtk_block_for_gc is a barrier point,
+ * one or more Ractors could be paused there. However, mmtk_before_fork is
+ * not compatible with that because it assumes that the MMTk workers are idle,
+ * but the workers are not idle because they are busy working on a GC.
+ *
+ * This essentially implements a trylock. It will optimistically lock but will
+ * release the lock if it detects that any other Ractors are waiting in
+ * rb_mmtk_block_for_gc.
+ */
+ rb_atomic_t mutator_blocking_count = RUBY_ATOMIC_LOAD(objspace->mutator_blocking_count);
+ if (mutator_blocking_count != 0) {
+ RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev);
+ goto retry;
}
- if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(objspace->next_object_id))) {
- rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
- }
- else {
- rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10)));
- }
-}
-
-// Forking
-
-void
-rb_gc_impl_before_fork(void *objspace_ptr)
-{
mmtk_before_fork();
}
void
rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
{
+ struct objspace *objspace = objspace_ptr;
+
mmtk_after_fork(rb_gc_get_ractor_newobj_cache());
+
+ RB_GC_VM_UNLOCK(objspace->fork_hook_vm_lock_lev);
}
// Statistics
@@ -1235,6 +1489,7 @@ rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE hash_or_key)
enum gc_stat_sym {
gc_stat_sym_count,
+ gc_stat_sym_moving_gc_count,
gc_stat_sym_time,
gc_stat_sym_total_allocated_objects,
gc_stat_sym_total_bytes,
@@ -1242,6 +1497,7 @@ enum gc_stat_sym {
gc_stat_sym_free_bytes,
gc_stat_sym_starting_heap_address,
gc_stat_sym_last_heap_address,
+ gc_stat_sym_weak_references_count,
gc_stat_sym_last
};
@@ -1253,6 +1509,7 @@ setup_gc_stat_symbols(void)
if (gc_stat_symbols[0] == 0) {
#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
S(count);
+ S(moving_gc_count);
S(time);
S(total_allocated_objects);
S(total_bytes);
@@ -1260,6 +1517,7 @@ setup_gc_stat_symbols(void)
S(free_bytes);
S(starting_heap_address);
S(last_heap_address);
+ S(weak_references_count);
}
}
@@ -1288,6 +1546,7 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
SET(count, objspace->gc_count);
+ SET(moving_gc_count, objspace->moving_gc_count);
SET(time, objspace->total_gc_time / (1000 * 1000));
SET(total_allocated_objects, objspace->total_allocated_objects);
SET(total_bytes, mmtk_total_bytes());
@@ -1295,6 +1554,7 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
SET(free_bytes, mmtk_free_bytes());
SET(starting_heap_address, (size_t)mmtk_starting_heap_address());
SET(last_heap_address, (size_t)mmtk_last_heap_address());
+ SET(weak_references_count, mmtk_weak_references_count());
#undef SET
if (!NIL_P(key)) {
@@ -1308,24 +1568,55 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
VALUE
rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
{
+ if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) {
+ int heap_idx = FIX2INT(heap_name);
+ if (heap_idx < 0 || heap_idx >= MMTK_HEAP_COUNT) {
+ rb_raise(rb_eArgError, "size pool index out of range");
+ }
+
+ if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) {
+ return SIZET2NUM(heap_sizes[heap_idx]);
+ }
+
+ return Qundef;
+ }
+
if (RB_TYPE_P(hash_or_sym, T_HASH)) {
return hash_or_sym;
}
- else {
- return Qundef;
- }
+
+ return Qundef;
}
// Miscellaneous
-#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 0
+#define RB_GC_OBJECT_METADATA_ENTRY_COUNT 1
static struct rb_gc_object_metadata_entry object_metadata_entries[RB_GC_OBJECT_METADATA_ENTRY_COUNT + 1];
struct rb_gc_object_metadata_entry *
rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
{
- object_metadata_entries[0].name = 0;
- object_metadata_entries[0].val = 0;
+ static ID ID_object_id;
+
+ if (!ID_object_id) {
+#define I(s) ID_##s = rb_intern(#s);
+ I(object_id);
+#undef I
+ }
+
+ size_t n = 0;
+
+#define SET_ENTRY(na, v) do { \
+ MMTK_ASSERT(n <= RB_GC_OBJECT_METADATA_ENTRY_COUNT); \
+ object_metadata_entries[n].name = ID_##na; \
+ object_metadata_entries[n].val = v; \
+ n++; \
+} while (0)
+
+ if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
+
+ object_metadata_entries[n].name = 0;
+ object_metadata_entries[n].val = 0;
return object_metadata_entries;
}
diff --git a/gc/mmtk/mmtk.h b/gc/mmtk/mmtk.h
index 614b832af5..b11e2873e3 100644
--- a/gc/mmtk/mmtk.h
+++ b/gc/mmtk/mmtk.h
@@ -20,6 +20,11 @@ typedef void *MMTk_ObjectReference;
typedef void *MMTk_NullableObjectReference;
typedef uint32_t MMTk_AllocationSemantics;
+typedef struct MMTk_BumpPointer {
+ uintptr_t cursor;
+ uintptr_t limit;
+} MMTk_BumpPointer;
+
#define MMTk_OBJREF_OFFSET 8
@@ -28,7 +33,6 @@ typedef uint32_t MMTk_AllocationSemantics;
#define MMTk_GC_THREAD_KIND_WORKER 1
typedef struct MMTk_RubyBindingOptions {
- bool ractor_check_mode;
size_t suffix_size;
} MMTk_RubyBindingOptions;
@@ -55,22 +59,26 @@ typedef struct MMTk_RubyUpcalls {
void (*init_gc_worker_thread)(struct MMTk_GCThreadTLS *gc_worker_tls);
bool (*is_mutator)(void);
void (*stop_the_world)(void);
- void (*resume_mutators)(void);
+ void (*resume_mutators)(bool gc_may_move);
void (*block_for_gc)(MMTk_VMMutatorThread tls);
+ void (*before_updating_jit_code)(void);
+ void (*after_updating_jit_code)(void);
size_t (*number_of_mutators)(void);
void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data);
void (*scan_gc_roots)(void);
void (*scan_objspace)(void);
- void (*scan_roots_in_mutator_thread)(MMTk_VMMutatorThread mutator_tls,
- MMTk_VMWorkerThread worker_tls);
- void (*scan_object_ruby_style)(MMTk_ObjectReference object);
+ void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to);
+ void (*update_object_references)(MMTk_ObjectReference object);
void (*call_gc_mark_children)(MMTk_ObjectReference object);
+ void (*handle_weak_references)(MMTk_ObjectReference object, bool moving);
void (*call_obj_free)(MMTk_ObjectReference object);
size_t (*vm_live_bytes)(void);
- void (*update_global_tables)(int tbl_idx);
+ void (*update_global_tables)(int tbl_idx, bool moving);
int (*global_tables_count)(void);
void (*update_finalizer_table)(void);
- void (*update_obj_id_tables)(void);
+ bool (*special_const_p)(MMTk_ObjectReference object);
+ void (*mutator_thread_panic_handler)(void);
+ void (*gc_thread_panic_handler)(void);
} MMTk_RubyUpcalls;
typedef struct MMTk_RawVecOfObjRef {
@@ -86,14 +94,15 @@ bool mmtk_is_reachable(MMTk_ObjectReference object);
MMTk_Builder *mmtk_builder_default(void);
void mmtk_init_binding(MMTk_Builder *builder,
- const struct MMTk_RubyBindingOptions *_binding_options,
- const struct MMTk_RubyUpcalls *upcalls,
- MMTk_ObjectReference weak_reference_dead_value);
+ const struct MMTk_RubyBindingOptions *binding_options,
+ const struct MMTk_RubyUpcalls *upcalls);
void mmtk_initialize_collection(MMTk_VMThread tls);
MMTk_Mutator *mmtk_bind_mutator(MMTk_VMMutatorThread tls);
+MMTk_BumpPointer *mmtk_get_bump_pointer_allocator(MMTk_Mutator *m);
+
void mmtk_destroy_mutator(MMTk_Mutator *mutator);
void mmtk_handle_user_collection_request(MMTk_VMMutatorThread tls, bool force, bool exhaustive);
@@ -113,11 +122,17 @@ void mmtk_post_alloc(MMTk_Mutator *mutator,
size_t bytes,
MMTk_AllocationSemantics semantics);
-void mmtk_add_obj_free_candidate(MMTk_ObjectReference object);
+void mmtk_add_obj_free_candidates(const MMTk_ObjectReference *objects,
+ size_t count,
+ bool can_parallel_free);
+
+void mmtk_declare_weak_references(MMTk_ObjectReference object);
+
+bool mmtk_weak_references_alive_p(MMTk_ObjectReference object);
-void mmtk_mark_weak(MMTk_ObjectReference *ptr);
+size_t mmtk_weak_references_count(void);
-void mmtk_remove_weak(const MMTk_ObjectReference *ptr);
+void mmtk_register_pinning_obj(MMTk_ObjectReference obj);
void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object);
diff --git a/gc/mmtk/src/abi.rs b/gc/mmtk/src/abi.rs
index fc9c899743..30890e0853 100644
--- a/gc/mmtk/src/abi.rs
+++ b/gc/mmtk/src/abi.rs
@@ -1,8 +1,12 @@
use crate::api::RubyMutator;
+use crate::extra_assert;
use crate::Ruby;
use libc::c_int;
use mmtk::scheduler::GCWorker;
-use mmtk::util::{Address, ObjectReference, VMMutatorThread, VMWorkerThread};
+use mmtk::util::Address;
+use mmtk::util::ObjectReference;
+use mmtk::util::VMMutatorThread;
+use mmtk::util::VMWorkerThread;
// For the C binding
pub const OBJREF_OFFSET: usize = 8;
@@ -10,16 +14,38 @@ pub const MIN_OBJ_ALIGN: usize = 8; // Even on 32-bit machine. A Ruby object is
pub const GC_THREAD_KIND_WORKER: libc::c_int = 1;
-const HAS_MOVED_GIVTBL: usize = 1 << 63;
const HIDDEN_SIZE_MASK: usize = 0x0000FFFFFFFFFFFF;
-// Should keep in sync with C code.
-const RUBY_FL_EXIVAR: usize = 1 << 10;
-
// An opaque type for the C counterpart.
#[allow(non_camel_case_types)]
pub struct st_table;
+#[repr(C)]
+pub struct HiddenHeader {
+ pub prefix: usize,
+}
+
+impl HiddenHeader {
+ #[inline(always)]
+ pub fn is_sane(&self) -> bool {
+ self.prefix & !HIDDEN_SIZE_MASK == 0
+ }
+
+ #[inline(always)]
+ fn assert_sane(&self) {
+ extra_assert!(
+ self.is_sane(),
+ "Hidden header is corrupted: {:x}",
+ self.prefix
+ );
+ }
+
+ pub fn payload_size(&self) -> usize {
+ self.assert_sane();
+ self.prefix & HIDDEN_SIZE_MASK
+ }
+}
+
/// Provide convenient methods for accessing Ruby objects.
/// TODO: Wrap C functions in `RubyUpcalls` as Rust-friendly methods.
pub struct RubyObjectAccess {
@@ -47,32 +73,17 @@ impl RubyObjectAccess {
self.suffix_addr() + Self::suffix_size()
}
- fn hidden_field(&self) -> Address {
- self.obj_start()
- }
-
- fn load_hidden_field(&self) -> usize {
- unsafe { self.hidden_field().load::<usize>() }
+ fn hidden_header(&self) -> &'static HiddenHeader {
+ unsafe { self.obj_start().as_ref() }
}
- fn update_hidden_field<F>(&self, f: F)
- where
- F: FnOnce(usize) -> usize,
- {
- let old_value = self.load_hidden_field();
- let new_value = f(old_value);
- unsafe {
- self.hidden_field().store(new_value);
- }
+ #[allow(unused)] // Maybe we need to mutate the hidden header in the future.
+ fn hidden_header_mut(&self) -> &'static mut HiddenHeader {
+ unsafe { self.obj_start().as_mut_ref() }
}
pub fn payload_size(&self) -> usize {
- self.load_hidden_field() & HIDDEN_SIZE_MASK
- }
-
- pub fn set_payload_size(&self, size: usize) {
- debug_assert!((size & HIDDEN_SIZE_MASK) == size);
- self.update_hidden_field(|old| old & !HIDDEN_SIZE_MASK | size & HIDDEN_SIZE_MASK);
+ self.hidden_header().payload_size()
}
fn flags_field(&self) -> Address {
@@ -83,22 +94,6 @@ impl RubyObjectAccess {
unsafe { self.flags_field().load::<usize>() }
}
- pub fn has_exivar_flag(&self) -> bool {
- (self.load_flags() & RUBY_FL_EXIVAR) != 0
- }
-
- pub fn has_moved_givtbl(&self) -> bool {
- (self.load_hidden_field() & HAS_MOVED_GIVTBL) != 0
- }
-
- pub fn set_has_moved_givtbl(&self) {
- self.update_hidden_field(|old| old | HAS_MOVED_GIVTBL)
- }
-
- pub fn clear_has_moved_givtbl(&self) {
- self.update_hidden_field(|old| old & !HAS_MOVED_GIVTBL)
- }
-
pub fn prefix_size() -> usize {
// Currently, a hidden size field of word size is placed before each object.
OBJREF_OFFSET
@@ -232,7 +227,7 @@ impl GCThreadTLS {
/// Has undefined behavior if `ptr` is invalid.
pub unsafe fn check_cast(ptr: *mut GCThreadTLS) -> &'static mut GCThreadTLS {
assert!(!ptr.is_null());
- let result = &mut *ptr;
+ let result = unsafe { &mut *ptr };
debug_assert!({
let kind = result.kind;
kind == GC_THREAD_KIND_WORKER
@@ -247,7 +242,7 @@ impl GCThreadTLS {
/// Has undefined behavior if `ptr` is invalid.
pub unsafe fn from_vwt_check(vwt: VMWorkerThread) -> &'static mut GCThreadTLS {
let ptr = Self::from_vwt(vwt);
- Self::check_cast(ptr)
+ unsafe { Self::check_cast(ptr) }
}
#[allow(clippy::not_unsafe_ptr_arg_deref)] // `transmute` does not dereference pointer
@@ -283,7 +278,7 @@ impl RawVecOfObjRef {
///
/// This function turns raw pointer into a Vec without check.
pub unsafe fn into_vec(self) -> Vec<ObjectReference> {
- Vec::from_raw_parts(self.ptr, self.len, self.capa)
+ unsafe { Vec::from_raw_parts(self.ptr, self.len, self.capa) }
}
}
@@ -296,7 +291,6 @@ impl From<Vec<ObjectReference>> for RawVecOfObjRef {
#[repr(C)]
#[derive(Clone)]
pub struct RubyBindingOptions {
- pub ractor_check_mode: bool,
pub suffix_size: usize,
}
@@ -306,8 +300,10 @@ pub struct RubyUpcalls {
pub init_gc_worker_thread: extern "C" fn(gc_worker_tls: *mut GCThreadTLS),
pub is_mutator: extern "C" fn() -> bool,
pub stop_the_world: extern "C" fn(),
- pub resume_mutators: extern "C" fn(),
+ pub resume_mutators: extern "C" fn(gc_may_move: bool),
pub block_for_gc: extern "C" fn(tls: VMMutatorThread),
+ pub before_updating_jit_code: extern "C" fn(),
+ pub after_updating_jit_code: extern "C" fn(),
pub number_of_mutators: extern "C" fn() -> usize,
pub get_mutators: extern "C" fn(
visit_mutator: extern "C" fn(*mut RubyMutator, *mut libc::c_void),
@@ -315,16 +311,18 @@ pub struct RubyUpcalls {
),
pub scan_gc_roots: extern "C" fn(),
pub scan_objspace: extern "C" fn(),
- pub scan_roots_in_mutator_thread:
- extern "C" fn(mutator_tls: VMMutatorThread, worker_tls: VMWorkerThread),
- pub scan_object_ruby_style: extern "C" fn(object: ObjectReference),
+ pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference),
+ pub update_object_references: extern "C" fn(object: ObjectReference),
pub call_gc_mark_children: extern "C" fn(object: ObjectReference),
+ pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool),
pub call_obj_free: extern "C" fn(object: ObjectReference),
pub vm_live_bytes: extern "C" fn() -> usize,
- pub update_global_tables: extern "C" fn(tbl_idx: c_int),
+ pub update_global_tables: extern "C" fn(tbl_idx: c_int, moving: bool),
pub global_tables_count: extern "C" fn() -> c_int,
pub update_finalizer_table: extern "C" fn(),
- pub update_obj_id_tables: extern "C" fn(),
+ pub special_const_p: extern "C" fn(object: ObjectReference) -> bool,
+ pub mutator_thread_panic_handler: extern "C" fn(),
+ pub gc_thread_panic_handler: extern "C" fn(),
}
unsafe impl Sync for RubyUpcalls {}
diff --git a/gc/mmtk/src/api.rs b/gc/mmtk/src/api.rs
index c15996727e..c0540fe0c8 100644
--- a/gc/mmtk/src/api.rs
+++ b/gc/mmtk/src/api.rs
@@ -1,16 +1,28 @@
-use std::sync::atomic::Ordering;
+// Functions in this module are unsafe for one reason:
+// They are called by C functions and they need to pass raw pointers to Rust.
+#![allow(clippy::missing_safety_doc)]
+
+use mmtk::util::alloc::BumpPointer;
+use mmtk::util::alloc::ImmixAllocator;
+use mmtk::util::conversions;
use mmtk::util::options::PlanSelector;
+use std::str::FromStr;
+use std::sync::atomic::Ordering;
use crate::abi::RawVecOfObjRef;
use crate::abi::RubyBindingOptions;
use crate::abi::RubyUpcalls;
use crate::binding;
use crate::binding::RubyBinding;
+use crate::heap::CpuHeapTriggerConfig;
+use crate::heap::RubyHeapTriggerConfig;
+use crate::heap::CPU_HEAP_TRIGGER_CONFIG;
+use crate::heap::RUBY_HEAP_TRIGGER_CONFIG;
use crate::mmtk;
-use crate::Ruby;
-use crate::RubySlot;
use crate::utils::default_heap_max;
use crate::utils::parse_capacity;
+use crate::Ruby;
+use crate::RubySlot;
use mmtk::memory_manager;
use mmtk::memory_manager::mmtk_init;
use mmtk::util::constants::MIN_OBJECT_SIZE;
@@ -37,73 +49,139 @@ pub extern "C" fn mmtk_is_reachable(object: ObjectReference) -> bool {
// =============== Bootup ===============
-fn mmtk_builder_default_parse_threads() -> usize {
- let threads_str = std::env::var("MMTK_THREADS")
- .unwrap_or("0".to_string());
-
- threads_str
- .parse::<usize>()
- .unwrap_or_else(|_err| {
- eprintln!("[FATAL] Invalid MMTK_THREADS {}", threads_str);
+fn parse_env_var_with<T, F: FnOnce(&str) -> Option<T>>(key: &str, parse: F) -> Option<T> {
+ let val = match std::env::var(key) {
+ Ok(val) => val,
+ Err(std::env::VarError::NotPresent) => return None,
+ Err(std::env::VarError::NotUnicode(os_string)) => {
+ eprintln!("[FATAL] Invalid {key} {os_string:?}");
std::process::exit(1);
- })
-}
+ }
+ };
-fn mmtk_builder_default_parse_heap_min() -> usize {
- const DEFAULT_HEAP_MIN: usize = 1 << 20;
+ let parsed = parse(&val).unwrap_or_else(|| {
+ eprintln!("[FATAL] Invalid {key} {val}");
+ std::process::exit(1);
+ });
- let heap_min_str = std::env::var("MMTK_HEAP_MIN")
- .unwrap_or(DEFAULT_HEAP_MIN.to_string());
+ Some(parsed)
+}
- let size = parse_capacity(&heap_min_str, 0);
- if size == 0 {
- eprintln!("[FATAL] Invalid MMTK_HEAP_MIN {}", heap_min_str);
- std::process::exit(1);
- }
+fn parse_env_var<T: FromStr>(key: &str) -> Option<T> {
+ parse_env_var_with(key, |s| s.parse().ok())
+}
- size
+fn mmtk_builder_default_parse_threads() -> Option<usize> {
+ parse_env_var("MMTK_THREADS")
+}
+
+fn mmtk_builder_default_parse_heap_min() -> usize {
+ const DEFAULT_HEAP_MIN: usize = 1 << 20;
+ parse_env_var_with("MMTK_HEAP_MIN", parse_capacity).unwrap_or(DEFAULT_HEAP_MIN)
}
fn mmtk_builder_default_parse_heap_max() -> usize {
- let heap_max_str = std::env::var("MMTK_HEAP_MAX")
- .unwrap_or(default_heap_max().to_string());
+ parse_env_var_with("MMTK_HEAP_MAX", parse_capacity).unwrap_or_else(default_heap_max)
+}
- let size = parse_capacity(&heap_max_str, 0);
- if size == 0 {
- eprintln!("[FATAL] Invalid MMTK_HEAP_MAX {}", heap_max_str);
- std::process::exit(1);
- }
+fn parse_float_env_var(key: &str, default: f64, min: f64, max: f64) -> f64 {
+ parse_env_var_with(key, |s| {
+ let mut float = f64::from_str(s).unwrap_or(default);
+
+ if float <= min {
+ eprintln!(
+ "{key} has value {float} which must be greater than {min}, using default instead"
+ );
+ float = default;
+ }
- size
+ if float >= max {
+ eprintln!(
+ "{key} has value {float} which must be less than {max}, using default instead"
+ );
+ float = default;
+ }
+
+ Some(float)
+ })
+ .unwrap_or(default)
}
fn mmtk_builder_default_parse_heap_mode(heap_min: usize, heap_max: usize) -> GCTriggerSelector {
- let heap_mode_str = std::env::var("MMTK_HEAP_MODE")
- .unwrap_or("dynamic".to_string());
-
- match heap_mode_str.as_str() {
- "fixed" => GCTriggerSelector::FixedHeapSize(heap_max),
- "dynamic" => GCTriggerSelector::DynamicHeapSize(heap_min, heap_max),
- _ => {
- eprintln!("[FATAL] Invalid MMTK_HEAP_MODE {}", heap_mode_str);
- std::process::exit(1);
+ let make_fixed = || GCTriggerSelector::FixedHeapSize(heap_max);
+ let make_dynamic = || GCTriggerSelector::DynamicHeapSize(heap_min, heap_max);
+
+ parse_env_var_with("MMTK_HEAP_MODE", |s| match s {
+ "fixed" => Some(make_fixed()),
+ "dynamic" => Some(make_dynamic()),
+ "ruby" => {
+ let min_ratio = parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", 0.2, 0.0, 1.0);
+ let goal_ratio =
+ parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", 0.4, min_ratio, 1.0);
+ let max_ratio =
+ parse_float_env_var("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", 0.65, goal_ratio, 1.0);
+
+ crate::heap::RUBY_HEAP_TRIGGER_CONFIG
+ .set(RubyHeapTriggerConfig {
+ min_heap_pages: conversions::bytes_to_pages_up(heap_min),
+ max_heap_pages: conversions::bytes_to_pages_up(heap_max),
+ heap_pages_min_ratio: min_ratio,
+ heap_pages_goal_ratio: goal_ratio,
+ heap_pages_max_ratio: max_ratio,
+ })
+ .unwrap_or_else(|_| panic!("RUBY_HEAP_TRIGGER_CONFIG is already set"));
+
+ Some(GCTriggerSelector::Delegated)
}
- }
+ "cpu" => {
+ // CPU-overhead-driven heap sizing based on Tavakolisomeh et al.,
+ // "Heap Size Adjustment with CPU Control", MPLR '23.
+ //
+ // Target is expressed as a percentage (0, 100) via
+ // `MMTK_GC_CPU_TARGET`. The paper recommends 15 for ZGC (a
+ // concurrent collector); we default to 5 for MMTk-Ruby. With
+ // MMTk's stop-the-world Immix, every percent of GC CPU is also
+ // a percent of wall-clock the mutator is blocked on, so a much
+ // smaller budget is appropriate. An empirical sweep across
+ // ruby-bench (railsbench, lobsters, psych-load, liquid-render,
+ // lee) found target=5 to be Pareto-optimal: ~6% geomean speedup
+ // vs. the `ruby` heap mode with effectively identical geomean
+ // peak RSS.
+ let target_percent = parse_float_env_var("MMTK_GC_CPU_TARGET", 5.0, 0.0, 100.0);
+ let window_size = parse_env_var::<usize>("MMTK_GC_CPU_WINDOW").unwrap_or(3);
+ let window_size = window_size.max(1);
+
+ let min_heap_pages = conversions::bytes_to_pages_up(heap_min);
+ let max_heap_pages = conversions::bytes_to_pages_up(heap_max);
+ // Start at the min heap size, as the other delegated triggers do.
+ // The control loop will adjust from here after the first GC cycle.
+ let initial_heap_pages = min_heap_pages;
+
+ CPU_HEAP_TRIGGER_CONFIG
+ .set(CpuHeapTriggerConfig {
+ min_heap_pages,
+ max_heap_pages,
+ initial_heap_pages,
+ target_gc_cpu: target_percent / 100.0,
+ window_size,
+ })
+ .unwrap_or_else(|_| panic!("CPU_HEAP_TRIGGER_CONFIG is already set"));
+
+ Some(GCTriggerSelector::Delegated)
+ }
+ _ => None,
+ })
+ .unwrap_or_else(make_dynamic)
}
fn mmtk_builder_default_parse_plan() -> PlanSelector {
- let plan_str = std::env::var("MMTK_PLAN")
- .unwrap_or("Immix".to_string());
-
- match plan_str.as_str() {
- "NoGC" => PlanSelector::NoGC,
- "MarkSweep" => PlanSelector::MarkSweep,
- "Immix" => PlanSelector::Immix,
- _ => {
- eprintln!("[FATAL] Invalid MMTK_PLAN {}", plan_str);
- std::process::exit(1);
- }
- }
+ parse_env_var_with("MMTK_PLAN", |s| match s {
+ "NoGC" => Some(PlanSelector::NoGC),
+ "MarkSweep" => Some(PlanSelector::MarkSweep),
+ "Immix" => Some(PlanSelector::Immix),
+ _ => None,
+ })
+ .unwrap_or(PlanSelector::Immix)
}
#[no_mangle]
@@ -111,9 +189,12 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
let mut builder = MMTKBuilder::new_no_env_vars();
builder.options.no_finalizer.set(true);
- let threads = mmtk_builder_default_parse_threads();
- if threads > 0 {
- builder.options.threads.set(threads);
+ if let Some(threads) = mmtk_builder_default_parse_threads() {
+ if !builder.options.threads.set(threads) {
+ // MMTk will validate it and reject 0.
+ eprintln!("[FATAL] Failed to set the number of MMTk threads to {threads}");
+ std::process::exit(1);
+ }
}
let heap_min = mmtk_builder_default_parse_heap_min();
@@ -121,11 +202,14 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
let heap_max = mmtk_builder_default_parse_heap_max();
if heap_min >= heap_max {
- eprintln!("[FATAL] MMTK_HEAP_MIN({}) >= MMTK_HEAP_MAX({})", heap_min, heap_max);
+ eprintln!("[FATAL] MMTK_HEAP_MIN({heap_min}) >= MMTK_HEAP_MAX({heap_max})");
std::process::exit(1);
}
- builder.options.gc_trigger.set(mmtk_builder_default_parse_heap_mode(heap_min, heap_max));
+ builder
+ .options
+ .gc_trigger
+ .set(mmtk_builder_default_parse_heap_mode(heap_min, heap_max));
builder.options.plan.set(mmtk_builder_default_parse_plan());
@@ -133,20 +217,26 @@ pub extern "C" fn mmtk_builder_default() -> *mut MMTKBuilder {
}
#[no_mangle]
-pub extern "C" fn mmtk_init_binding(
+pub unsafe extern "C" fn mmtk_init_binding(
builder: *mut MMTKBuilder,
- _binding_options: *const RubyBindingOptions,
+ binding_options: *const RubyBindingOptions,
upcalls: *const RubyUpcalls,
- weak_reference_dead_value: ObjectReference,
) {
+ crate::MUTATOR_THREAD_PANIC_HANDLER
+ .set((unsafe { (*upcalls).clone() }).mutator_thread_panic_handler)
+ .unwrap_or_else(|_| panic!("MUTATOR_THREAD_PANIC_HANDLER is already initialized"));
+
crate::set_panic_hook();
- let builder = unsafe { Box::from_raw(builder) };
- let binding_options = RubyBindingOptions {ractor_check_mode: false, suffix_size: 0};
+ let builder: Box<MMTKBuilder> = unsafe { Box::from_raw(builder) };
+ let binding_options = unsafe { (*binding_options).clone() };
let mmtk_boxed = mmtk_init(&builder);
let mmtk_static = Box::leak(Box::new(mmtk_boxed));
- let binding = RubyBinding::new(mmtk_static, &binding_options, upcalls, weak_reference_dead_value);
+ let mut binding = RubyBinding::new(mmtk_static, &binding_options, upcalls);
+ binding
+ .weak_proc
+ .init_parallel_obj_free_candidates(memory_manager::num_of_workers(binding.mmtk));
crate::BINDING
.set(binding)
@@ -164,7 +254,25 @@ pub extern "C" fn mmtk_bind_mutator(tls: VMMutatorThread) -> *mut RubyMutator {
}
#[no_mangle]
-pub extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) {
+pub unsafe extern "C" fn mmtk_get_bump_pointer_allocator(m: *mut RubyMutator) -> *mut BumpPointer {
+ match *crate::BINDING.get().unwrap().mmtk.get_options().plan {
+ PlanSelector::Immix => {
+ let mutator: &mut Mutator<Ruby> = unsafe { &mut *m };
+ let allocator =
+ unsafe { mutator.allocator_mut(mmtk::util::alloc::AllocatorSelector::Immix(0)) };
+
+ if let Some(immix_allocator) = allocator.downcast_mut::<ImmixAllocator<Ruby>>() {
+ &mut immix_allocator.bump_pointer as *mut BumpPointer
+ } else {
+ panic!("Failed to get bump pointer allocator");
+ }
+ }
+ _ => std::ptr::null_mut(),
+ }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn mmtk_destroy_mutator(mutator: *mut RubyMutator) {
// notify mmtk-core about destroyed mutator
memory_manager::destroy_mutator(unsafe { &mut *mutator });
// turn the ptr back to a box, and let Rust properly reclaim it
@@ -184,7 +292,9 @@ pub extern "C" fn mmtk_handle_user_collection_request(
#[no_mangle]
pub extern "C" fn mmtk_set_gc_enabled(enable: bool) {
- crate::CONFIGURATION.gc_enabled.store(enable, Ordering::Relaxed);
+ crate::CONFIGURATION
+ .gc_enabled
+ .store(enable, Ordering::Relaxed);
}
#[no_mangle]
@@ -195,7 +305,7 @@ pub extern "C" fn mmtk_gc_enabled_p() -> bool {
// =============== Object allocation ===============
#[no_mangle]
-pub extern "C" fn mmtk_alloc(
+pub unsafe extern "C" fn mmtk_alloc(
mutator: *mut RubyMutator,
size: usize,
align: usize,
@@ -213,7 +323,7 @@ pub extern "C" fn mmtk_alloc(
}
#[no_mangle]
-pub extern "C" fn mmtk_post_alloc(
+pub unsafe extern "C" fn mmtk_post_alloc(
mutator: *mut RubyMutator,
refer: ObjectReference,
bytes: usize,
@@ -222,28 +332,46 @@ pub extern "C" fn mmtk_post_alloc(
memory_manager::post_alloc::<Ruby>(unsafe { &mut *mutator }, refer, bytes, semantics)
}
-// TODO: Replace with buffered mmtk_add_obj_free_candidates
#[no_mangle]
-pub extern "C" fn mmtk_add_obj_free_candidate(object: ObjectReference) {
- binding().weak_proc.add_obj_free_candidate(object)
+pub unsafe extern "C" fn mmtk_add_obj_free_candidates(
+ objects: *const ObjectReference,
+ count: usize,
+ can_parallel_free: bool,
+) {
+ let objects = unsafe { std::slice::from_raw_parts(objects, count) };
+ binding()
+ .weak_proc
+ .add_obj_free_candidates_batch(objects, can_parallel_free)
+}
+
+// =============== Weak references ===============
+
+#[no_mangle]
+pub extern "C" fn mmtk_declare_weak_references(object: ObjectReference) {
+ binding().weak_proc.add_weak_reference(object);
}
-// =============== Marking ===============
+#[no_mangle]
+pub extern "C" fn mmtk_weak_references_alive_p(object: ObjectReference) -> bool {
+ object.is_reachable()
+}
#[no_mangle]
-pub extern "C" fn mmtk_mark_weak(ptr: &'static mut ObjectReference) {
- binding().weak_proc.add_weak_reference(ptr);
+pub extern "C" fn mmtk_weak_references_count() -> usize {
+ binding().weak_proc.weak_references_count()
}
+// =============== Compaction ===============
+
#[no_mangle]
-pub extern "C" fn mmtk_remove_weak(ptr: &ObjectReference) {
- binding().weak_proc.remove_weak_reference(ptr);
+pub extern "C" fn mmtk_register_pinning_obj(obj: ObjectReference) {
+ crate::binding().pinning_registry.register(obj);
}
// =============== Write barriers ===============
#[no_mangle]
-pub extern "C" fn mmtk_object_reference_write_post(
+pub unsafe extern "C" fn mmtk_object_reference_write_post(
mutator: *mut RubyMutator,
object: ObjectReference,
) {
@@ -347,7 +475,7 @@ pub extern "C" fn mmtk_plan() -> *const u8 {
PlanSelector::NoGC => NO_GC.as_ptr(),
PlanSelector::MarkSweep => MARK_SWEEP.as_ptr(),
PlanSelector::Immix => IMMIX.as_ptr(),
- _ => panic!("Unknown plan")
+ _ => panic!("Unknown plan"),
}
}
@@ -355,11 +483,21 @@ pub extern "C" fn mmtk_plan() -> *const u8 {
pub extern "C" fn mmtk_heap_mode() -> *const u8 {
static FIXED_HEAP: &[u8] = b"fixed\0";
static DYNAMIC_HEAP: &[u8] = b"dynamic\0";
+ static RUBY_HEAP: &[u8] = b"ruby\0";
+ static CPU_HEAP: &[u8] = b"cpu\0";
match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger {
GCTriggerSelector::FixedHeapSize(_) => FIXED_HEAP.as_ptr(),
GCTriggerSelector::DynamicHeapSize(_, _) => DYNAMIC_HEAP.as_ptr(),
- _ => panic!("Unknown heap mode")
+ GCTriggerSelector::Delegated => {
+ // Two delegated triggers exist; disambiguate via the populated
+ // config singleton.
+ if CPU_HEAP_TRIGGER_CONFIG.get().is_some() {
+ CPU_HEAP.as_ptr()
+ } else {
+ RUBY_HEAP.as_ptr()
+ }
+ }
}
}
@@ -368,7 +506,18 @@ pub extern "C" fn mmtk_heap_min() -> usize {
match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger {
GCTriggerSelector::FixedHeapSize(_) => 0,
GCTriggerSelector::DynamicHeapSize(min_size, _) => min_size,
- _ => panic!("Unknown heap mode")
+ GCTriggerSelector::Delegated => {
+ if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() {
+ conversions::pages_to_bytes(cfg.min_heap_pages)
+ } else {
+ conversions::pages_to_bytes(
+ RUBY_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("RUBY_HEAP_TRIGGER_CONFIG not set")
+ .min_heap_pages,
+ )
+ }
+ }
}
}
@@ -377,7 +526,18 @@ pub extern "C" fn mmtk_heap_max() -> usize {
match *crate::BINDING.get().unwrap().mmtk.get_options().gc_trigger {
GCTriggerSelector::FixedHeapSize(max_size) => max_size,
GCTriggerSelector::DynamicHeapSize(_, max_size) => max_size,
- _ => panic!("Unknown heap mode")
+ GCTriggerSelector::Delegated => {
+ if let Some(cfg) = CPU_HEAP_TRIGGER_CONFIG.get() {
+ conversions::pages_to_bytes(cfg.max_heap_pages)
+ } else {
+ conversions::pages_to_bytes(
+ RUBY_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("RUBY_HEAP_TRIGGER_CONFIG not set")
+ .max_heap_pages,
+ )
+ }
+ }
}
}
diff --git a/gc/mmtk/src/binding.rs b/gc/mmtk/src/binding.rs
index e0f8640e1c..36d4a992fd 100644
--- a/gc/mmtk/src/binding.rs
+++ b/gc/mmtk/src/binding.rs
@@ -9,6 +9,7 @@ use mmtk::MMTK;
use crate::abi;
use crate::abi::RubyBindingOptions;
+use crate::pinning_registry::PinningRegistry;
use crate::weak_proc::WeakProcessor;
use crate::Ruby;
@@ -54,10 +55,9 @@ pub struct RubyBinding {
pub upcalls: *const abi::RubyUpcalls,
pub plan_name: Mutex<Option<CString>>,
pub weak_proc: WeakProcessor,
+ pub pinning_registry: PinningRegistry,
pub gc_thread_join_handles: Mutex<Vec<JoinHandle<()>>>,
pub wb_unprotected_objects: Mutex<HashSet<ObjectReference>>,
-
- pub weak_reference_dead_value: ObjectReference,
}
unsafe impl Sync for RubyBinding {}
@@ -68,7 +68,6 @@ impl RubyBinding {
mmtk: &'static MMTK<Ruby>,
binding_options: &RubyBindingOptions,
upcalls: *const abi::RubyUpcalls,
- weak_reference_dead_value: ObjectReference,
) -> Self {
unsafe {
crate::BINDING_FAST.suffix_size = binding_options.suffix_size;
@@ -80,10 +79,9 @@ impl RubyBinding {
upcalls,
plan_name: Mutex::new(None),
weak_proc: WeakProcessor::new(),
+ pinning_registry: PinningRegistry::new(),
gc_thread_join_handles: Default::default(),
wb_unprotected_objects: Default::default(),
-
- weak_reference_dead_value
}
}
@@ -119,7 +117,7 @@ impl RubyBinding {
}
pub fn register_wb_unprotected_object(&self, object: ObjectReference) {
- debug!("Registering WB-unprotected object: {}", object);
+ debug!("Registering WB-unprotected object: {object}");
let mut objects = self.wb_unprotected_objects.lock().unwrap();
objects.insert(object);
}
diff --git a/gc/mmtk/src/collection.rs b/gc/mmtk/src/collection.rs
index 0570b64e3a..648efa4e27 100644
--- a/gc/mmtk/src/collection.rs
+++ b/gc/mmtk/src/collection.rs
@@ -1,14 +1,26 @@
use crate::abi::GCThreadTLS;
use crate::api::RubyMutator;
-use crate::{mmtk, upcalls, Ruby};
+use crate::heap::CpuHeapTrigger;
+use crate::heap::RubyHeapTrigger;
+use crate::heap::CPU_HEAP_TRIGGER_CONFIG;
+use crate::mmtk;
+use crate::upcalls;
+use crate::Ruby;
use mmtk::memory_manager;
use mmtk::scheduler::*;
-use mmtk::util::{VMMutatorThread, VMThread, VMWorkerThread};
-use mmtk::vm::{Collection, GCThreadContext};
+use mmtk::util::heap::GCTriggerPolicy;
+use mmtk::util::VMMutatorThread;
+use mmtk::util::VMThread;
+use mmtk::util::VMWorkerThread;
+use mmtk::vm::Collection;
+use mmtk::vm::GCThreadContext;
+use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::thread;
+static CURRENT_GC_MAY_MOVE: AtomicBool = AtomicBool::new(false);
+
pub struct VMCollection {}
impl Collection<Ruby> for VMCollection {
@@ -16,11 +28,21 @@ impl Collection<Ruby> for VMCollection {
crate::CONFIGURATION.gc_enabled.load(Ordering::Relaxed)
}
- fn stop_all_mutators<F>(_tls: VMWorkerThread, mut mutator_visitor: F)
+ fn stop_all_mutators<F>(tls: VMWorkerThread, mut mutator_visitor: F)
where
F: FnMut(&'static mut mmtk::Mutator<Ruby>),
{
(upcalls().stop_the_world)();
+
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ CURRENT_GC_MAY_MOVE.store(true, Ordering::Relaxed);
+ (upcalls().before_updating_jit_code)();
+ } else {
+ CURRENT_GC_MAY_MOVE.store(false, Ordering::Relaxed);
+ }
+
+ crate::binding().pinning_registry.pin_children(tls);
+
(upcalls().get_mutators)(
Self::notify_mutator_ready::<F>,
&mut mutator_visitor as *mut F as *mut _,
@@ -28,7 +50,13 @@ impl Collection<Ruby> for VMCollection {
}
fn resume_mutators(_tls: VMWorkerThread) {
- (upcalls().resume_mutators)();
+ let current_gc_may_move = CURRENT_GC_MAY_MOVE.load(Ordering::Relaxed);
+
+ if current_gc_may_move {
+ (upcalls().after_updating_jit_code)();
+ }
+
+ (upcalls().resume_mutators)(current_gc_may_move);
}
fn block_for_gc(tls: VMMutatorThread) {
@@ -41,10 +69,7 @@ impl Collection<Ruby> for VMCollection {
.name("MMTk Worker Thread".to_string())
.spawn(move || {
let ordinal = worker.ordinal;
- debug!(
- "Hello! This is MMTk Worker Thread running! ordinal: {}",
- ordinal
- );
+ debug!("Hello! This is MMTk Worker Thread running! ordinal: {ordinal}");
crate::register_gc_thread(thread::current().id());
let ptr_worker = &mut *worker as *mut GCWorker<Ruby>;
let gc_thread_tls =
@@ -55,10 +80,7 @@ impl Collection<Ruby> for VMCollection {
GCThreadTLS::to_vwt(gc_thread_tls),
worker,
);
- debug!(
- "An MMTk Worker Thread is quitting. Good bye! ordinal: {}",
- ordinal
- );
+ debug!("An MMTk Worker Thread is quitting. Good bye! ordinal: {ordinal}");
crate::unregister_gc_thread(thread::current().id());
})
.unwrap(),
@@ -73,6 +95,19 @@ impl Collection<Ruby> for VMCollection {
fn vm_live_bytes() -> usize {
(upcalls().vm_live_bytes)()
}
+
+ fn create_gc_trigger() -> Box<dyn GCTriggerPolicy<Ruby>> {
+ // `GCTriggerSelector::Delegated` is currently used by two different
+ // heap modes: `ruby` (the Ruby-like free-slot ratio trigger) and `cpu`
+ // (the CPU-overhead trigger from Tavakolisomeh et al., MPLR '23).
+ // Which one is active is determined by which `OnceCell` config the
+ // `MMTK_HEAP_MODE` parser populated.
+ if CPU_HEAP_TRIGGER_CONFIG.get().is_some() {
+ Box::new(CpuHeapTrigger::default())
+ } else {
+ Box::new(RubyHeapTrigger::default())
+ }
+ }
}
impl VMCollection {
diff --git a/gc/mmtk/src/heap/cpu_heap_trigger.rs b/gc/mmtk/src/heap/cpu_heap_trigger.rs
new file mode 100644
index 0000000000..ef5a79fe9a
--- /dev/null
+++ b/gc/mmtk/src/heap/cpu_heap_trigger.rs
@@ -0,0 +1,370 @@
+//! A GC trigger that adjusts the heap size based on the CPU overhead of GC.
+//!
+//! This is an implementation of the heap sizing policy described in
+//! Tavakolisomeh, Shimchenko, Österlund, Bruno, Ferreira, Wrigstad,
+//! "Heap Size Adjustment with CPU Control", MPLR '23.
+//! <https://doi.org/10.1145/3617651.3622988>
+//!
+//! The idea: rather than letting heap size control GC frequency, let a
+//! user-supplied *target GC CPU overhead* control the heap size. After each GC
+//! cycle, we measure the GC CPU overhead (fraction of process CPU time spent
+//! in GC) and compare it to the target. If GC is over budget we grow the heap
+//! (reducing GC frequency); if it is under budget we shrink the heap (trading
+//! memory for more frequent collections).
+//!
+//! ## Algorithm
+//!
+//! After each GC cycle we compute, using an average of the last `n` cycles:
+//!
+//! ```text
+//! GC_CPU = T_GC / T_APP (Eq. 1)
+//! overhead_error = GC_CPU - target (Eq. 2)
+//! sigmoid_error = 1 / (1 + e^(-overhead_error)) (Eq. 3)
+//! adjustment_factor = sigmoid_error + 0.5 (in (0.5, 1.5)) (Eq. 4)
+//! new_size = current_size * adjustment_factor (Eq. 5)
+//! ```
+//!
+//! where:
+//! - `T_GC` is the wall-clock duration of each GC cycle.
+//! - `T_APP` is process CPU time elapsed between consecutive GC cycles (sum of
+//! CPU time over all threads — mutators, GC workers, compilers, etc.), read
+//! via `clock_gettime(CLOCK_PROCESS_CPUTIME_ID)`.
+//!
+//! The final heap size is then clamped to the range
+//! `[max(1.1 * used, min_heap_pages), max_heap_pages]`, providing 10% headroom
+//! above current live memory to avoid triggering GC on an effectively-empty
+//! heap.
+//!
+//! ## Differences from the paper
+//!
+//! The paper targets ZGC, a concurrent generational collector. MMTk's Ruby
+//! binding currently ships stop-the-world collectors (Immix, MarkSweep). The
+//! paper's formula still applies: with a STW collector the process CPU time
+//! during GC closely tracks the wall-clock GC time, and mutator CPU time
+//! during the mutator phase is correctly attributed. For generational plans
+//! we skip nursery-only GCs, consistent with MemBalancer.
+
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+use std::sync::Mutex;
+
+use mmtk::util::heap::GCTriggerPolicy;
+use mmtk::util::heap::SpaceStats;
+use mmtk::Plan;
+use mmtk::MMTK;
+use once_cell::sync::OnceCell;
+
+use crate::Ruby;
+
+pub static CPU_HEAP_TRIGGER_CONFIG: OnceCell<CpuHeapTriggerConfig> = OnceCell::new();
+
+/// Configuration for the [`CpuHeapTrigger`].
+pub struct CpuHeapTriggerConfig {
+ /// Lower bound on heap size (in pages). The trigger will never shrink below
+ /// this value.
+ pub min_heap_pages: usize,
+ /// Upper bound on heap size (in pages). The trigger will never grow above
+ /// this value.
+ pub max_heap_pages: usize,
+ /// Initial heap size (in pages).
+ pub initial_heap_pages: usize,
+ /// Target GC CPU overhead as a fraction of total process CPU time. For
+ /// example, `0.15` means the policy will try to keep GC CPU usage near 15%.
+ /// Valid range: `(0.0, 1.0)`.
+ pub target_gc_cpu: f64,
+ /// Number of recent GC cycles averaged together when computing the CPU
+ /// overhead signal. Smoothes out short-term fluctuations. The paper uses 3.
+ pub window_size: usize,
+}
+
+/// A single GC cycle's timing measurements.
+#[derive(Clone, Copy, Debug, Default)]
+struct GcSample {
+ /// Wall-clock seconds spent inside this GC cycle.
+ gc_seconds: f64,
+ /// Seconds of process CPU time elapsed since the previous GC cycle ended.
+ /// This covers both mutator time and (on multi-threaded mutators) any
+ /// mutator CPU time consumed in parallel with the previous GC.
+ app_cpu_seconds: f64,
+}
+
+struct CpuHeapTriggerState {
+ /// Ring buffer of the last `window_size` samples. Oldest-first.
+ samples: Vec<GcSample>,
+ /// Wall-clock time when the current GC cycle started. `None` when no GC is
+ /// in progress.
+ gc_start_wall: Option<std::time::Instant>,
+ /// Process CPU time (seconds) recorded at the end of the previous GC
+ /// cycle. `None` until the first cycle completes.
+ last_gc_end_cpu: Option<f64>,
+}
+
+impl CpuHeapTriggerState {
+ fn new() -> Self {
+ Self {
+ samples: Vec::new(),
+ gc_start_wall: None,
+ last_gc_end_cpu: None,
+ }
+ }
+
+ /// Pushes a new sample, dropping the oldest when the window is full.
+ fn push_sample(&mut self, sample: GcSample, window_size: usize) {
+ if self.samples.len() >= window_size {
+ self.samples.remove(0);
+ }
+ self.samples.push(sample);
+ }
+
+ /// Returns the arithmetic mean GC CPU overhead across the window, or
+ /// `None` if we don't yet have a full sample (which happens on the first
+ /// GC cycle — we have no baseline for `app_cpu_seconds`).
+ fn mean_gc_cpu(&self) -> Option<f64> {
+ if self.samples.is_empty() {
+ return None;
+ }
+ let total_gc: f64 = self.samples.iter().map(|s| s.gc_seconds).sum();
+ let total_app: f64 = self.samples.iter().map(|s| s.app_cpu_seconds).sum();
+ if total_app <= 0.0 {
+ return None;
+ }
+ Some(total_gc / total_app)
+ }
+}
+
+pub struct CpuHeapTrigger {
+ /// Target heap size in pages. Updated at the end of each GC cycle.
+ target_heap_pages: AtomicUsize,
+ /// Mutable timing state. Wrapped in a `Mutex` because `on_gc_start` and
+ /// `on_gc_end` are the only mutation sites and they are not on an
+ /// allocation hot path; avoiding the complexity of lock-free state is
+ /// worth the trivial contention.
+ state: Mutex<CpuHeapTriggerState>,
+}
+
+impl Default for CpuHeapTrigger {
+ fn default() -> Self {
+ let cfg = Self::get_config();
+ Self {
+ target_heap_pages: AtomicUsize::new(cfg.initial_heap_pages),
+ state: Mutex::new(CpuHeapTriggerState::new()),
+ }
+ }
+}
+
+impl GCTriggerPolicy<Ruby> for CpuHeapTrigger {
+ fn is_gc_required(
+ &self,
+ space_full: bool,
+ space: Option<SpaceStats<Ruby>>,
+ plan: &dyn Plan<VM = Ruby>,
+ ) -> bool {
+ // Let the plan decide, matching the other triggers.
+ plan.collection_required(space_full, space)
+ }
+
+ fn on_gc_start(&self, _mmtk: &'static MMTK<Ruby>) {
+ let mut state = self.state.lock().unwrap();
+ state.gc_start_wall = Some(std::time::Instant::now());
+ }
+
+ fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) {
+ // Skip nursery-only GCs for generational plans. The heap resizing
+ // decision is driven by the (much more expensive) full collections
+ // where the signal-to-noise ratio is high enough to be useful.
+ if let Some(gen_plan) = mmtk.get_plan().generational() {
+ if gen_plan.is_current_gc_nursery() {
+ return;
+ }
+ }
+
+ let cfg = Self::get_config();
+ let gc_end_cpu = process_cpu_time_seconds();
+
+ let mut state = self.state.lock().unwrap();
+
+ // Duration of this GC cycle (wall clock).
+ let gc_seconds = state
+ .gc_start_wall
+ .take()
+ .map(|start| start.elapsed().as_secs_f64())
+ .unwrap_or(0.0);
+
+ // Process CPU time elapsed since the previous GC cycle ended. We
+ // require at least one previous end timestamp to produce a valid
+ // sample — without it we cannot compute `T_APP`.
+ if let (Some(last_end), Some(now)) = (state.last_gc_end_cpu, gc_end_cpu) {
+ let app_cpu_seconds = (now - last_end).max(0.0);
+ // Only record non-degenerate samples to avoid poisoning the window
+ // with zero-time entries from back-to-back GCs.
+ if app_cpu_seconds > 0.0 {
+ state.push_sample(
+ GcSample {
+ gc_seconds,
+ app_cpu_seconds,
+ },
+ cfg.window_size,
+ );
+ }
+ }
+ state.last_gc_end_cpu = gc_end_cpu;
+
+ // Compute the new heap size only when we have samples to average over.
+ if let Some(gc_cpu) = state.mean_gc_cpu() {
+ // Drop the lock before doing the (relatively cheap) math and
+ // atomic update; nothing below needs the state.
+ drop(state);
+
+ let overhead_error = gc_cpu - cfg.target_gc_cpu; // Eq. (2)
+ let sigmoid_error = sigmoid(overhead_error); // Eq. (3)
+ let adjustment_factor = sigmoid_error + 0.5; // Eq. (4), range (0.5, 1.5)
+
+ let current = self.target_heap_pages.load(Ordering::Relaxed);
+ let suggested = ((current as f64) * adjustment_factor) as usize; // Eq. (5)
+
+ // Clamp:
+ // - upper bound: configured max
+ // - lower bound: max(1.1 * used, min) — 10% headroom above current
+ // live memory, so we never request a heap so small that GC is
+ // triggered immediately on return from this one.
+ let used = mmtk.get_plan().get_used_pages();
+ let floor = ((used as f64) * 1.1).ceil() as usize;
+ let lower = floor.max(cfg.min_heap_pages).min(cfg.max_heap_pages);
+ let upper = cfg.max_heap_pages;
+ let new_target = suggested.clamp(lower, upper);
+
+ self.target_heap_pages.store(new_target, Ordering::Relaxed);
+
+ info!(
+ "CpuHeapTrigger: gc_cpu={:.4} target={:.4} factor={:.4} \
+ pages {} -> {} (used={}, clamp=[{}, {}])",
+ gc_cpu,
+ cfg.target_gc_cpu,
+ adjustment_factor,
+ current,
+ new_target,
+ used,
+ lower,
+ upper
+ );
+ }
+ }
+
+ fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool {
+ plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_current_heap_size_in_pages(&self) -> usize {
+ self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_max_heap_size_in_pages(&self) -> usize {
+ Self::get_config().max_heap_pages
+ }
+
+ fn can_heap_size_grow(&self) -> bool {
+ self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages
+ }
+}
+
+impl CpuHeapTrigger {
+ fn get_config<'b>() -> &'b CpuHeapTriggerConfig {
+ CPU_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("Attempt to use CPU_HEAP_TRIGGER_CONFIG before it is initialized")
+ }
+}
+
+/// Standard logistic sigmoid. Returns 0.5 when x == 0, asymptotes to 0 and 1.
+fn sigmoid(x: f64) -> f64 {
+ 1.0 / (1.0 + (-x).exp())
+}
+
+/// Reads the process-wide CPU time as a floating-point number of seconds,
+/// summed across all threads of this process. Returns `None` if the clock
+/// query fails (which should be essentially impossible on supported
+/// platforms).
+fn process_cpu_time_seconds() -> Option<f64> {
+ let mut ts = libc::timespec {
+ tv_sec: 0,
+ tv_nsec: 0,
+ };
+ // SAFETY: `clock_gettime` writes exactly `sizeof(timespec)` bytes to the
+ // pointer we pass, which is a valid local stack allocation.
+ let rc = unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) };
+ if rc != 0 {
+ return None;
+ }
+ Some((ts.tv_sec as f64) + (ts.tv_nsec as f64) / 1_000_000_000.0)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn sigmoid_is_well_behaved() {
+ assert!((sigmoid(0.0) - 0.5).abs() < 1e-12);
+ assert!(sigmoid(-100.0) < 1e-9);
+ assert!(sigmoid(100.0) > 1.0 - 1e-9);
+ // Monotonic.
+ assert!(sigmoid(-1.0) < sigmoid(0.0));
+ assert!(sigmoid(0.0) < sigmoid(1.0));
+ }
+
+ #[test]
+ fn adjustment_factor_is_within_paper_bounds() {
+ // Eq. (4): adjustment_factor = sigmoid(e) + 0.5 must lie in (0.5, 1.5).
+ for e in [-10.0_f64, -1.0, 0.0, 1.0, 10.0] {
+ let f = sigmoid(e) + 0.5;
+ assert!(f > 0.5 && f < 1.5, "factor {f} out of range for e={e}");
+ }
+ }
+
+ #[test]
+ fn mean_gc_cpu_is_total_weighted() {
+ let mut state = CpuHeapTriggerState::new();
+ state.push_sample(
+ GcSample {
+ gc_seconds: 1.0,
+ app_cpu_seconds: 10.0,
+ },
+ 3,
+ );
+ state.push_sample(
+ GcSample {
+ gc_seconds: 3.0,
+ app_cpu_seconds: 10.0,
+ },
+ 3,
+ );
+ // (1 + 3) / (10 + 10) = 0.2
+ assert!((state.mean_gc_cpu().unwrap() - 0.2).abs() < 1e-12);
+ }
+
+ #[test]
+ fn window_drops_oldest() {
+ let mut state = CpuHeapTriggerState::new();
+ for i in 0..5 {
+ state.push_sample(
+ GcSample {
+ gc_seconds: i as f64,
+ app_cpu_seconds: 1.0,
+ },
+ 3,
+ );
+ }
+ assert_eq!(state.samples.len(), 3);
+ // After pushing 0,1,2,3,4 with window 3, we should have [2,3,4].
+ assert_eq!(state.samples[0].gc_seconds, 2.0);
+ assert_eq!(state.samples[2].gc_seconds, 4.0);
+ }
+
+ #[test]
+ fn no_sample_without_prior_gc() {
+ // First GC cycle cannot produce a sample (no previous end time). The
+ // push happens only when last_gc_end_cpu is Some.
+ let state = CpuHeapTriggerState::new();
+ assert!(state.mean_gc_cpu().is_none());
+ }
+}
diff --git a/gc/mmtk/src/heap/mod.rs b/gc/mmtk/src/heap/mod.rs
new file mode 100644
index 0000000000..05a35efb23
--- /dev/null
+++ b/gc/mmtk/src/heap/mod.rs
@@ -0,0 +1,9 @@
+mod cpu_heap_trigger;
+mod ruby_heap_trigger;
+
+pub use cpu_heap_trigger::CpuHeapTrigger;
+pub use cpu_heap_trigger::CpuHeapTriggerConfig;
+pub use cpu_heap_trigger::CPU_HEAP_TRIGGER_CONFIG;
+pub use ruby_heap_trigger::RubyHeapTrigger;
+pub use ruby_heap_trigger::RubyHeapTriggerConfig;
+pub use ruby_heap_trigger::RUBY_HEAP_TRIGGER_CONFIG;
diff --git a/gc/mmtk/src/heap/ruby_heap_trigger.rs b/gc/mmtk/src/heap/ruby_heap_trigger.rs
new file mode 100644
index 0000000000..fe1130043d
--- /dev/null
+++ b/gc/mmtk/src/heap/ruby_heap_trigger.rs
@@ -0,0 +1,105 @@
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
+
+use mmtk::util::heap::GCTriggerPolicy;
+use mmtk::util::heap::SpaceStats;
+use mmtk::Plan;
+use mmtk::MMTK;
+use once_cell::sync::OnceCell;
+
+use crate::Ruby;
+
+pub static RUBY_HEAP_TRIGGER_CONFIG: OnceCell<RubyHeapTriggerConfig> = OnceCell::new();
+
+pub struct RubyHeapTriggerConfig {
+ /// Min heap size
+ pub min_heap_pages: usize,
+ /// Max heap size
+ pub max_heap_pages: usize,
+ /// Minimum ratio of empty space after a GC before the heap will grow
+ pub heap_pages_min_ratio: f64,
+ /// Ratio the heap will grow by
+ pub heap_pages_goal_ratio: f64,
+ /// Maximum ratio of empty space after a GC before the heap will shrink
+ pub heap_pages_max_ratio: f64,
+}
+
+pub struct RubyHeapTrigger {
+ /// Target number of heap pages
+ target_heap_pages: AtomicUsize,
+}
+
+impl GCTriggerPolicy<Ruby> for RubyHeapTrigger {
+ fn is_gc_required(
+ &self,
+ space_full: bool,
+ space: Option<SpaceStats<Ruby>>,
+ plan: &dyn Plan<VM = Ruby>,
+ ) -> bool {
+ // Let the plan decide
+ plan.collection_required(space_full, space)
+ }
+
+ fn on_gc_end(&self, mmtk: &'static MMTK<Ruby>) {
+ if let Some(plan) = mmtk.get_plan().generational() {
+ if plan.is_current_gc_nursery() {
+ // Nursery GC
+ } else {
+ // Full GC
+ }
+
+ panic!("TODO: support for generational GC not implemented")
+ } else {
+ let used_pages = mmtk.get_plan().get_used_pages();
+
+ let target_min =
+ (used_pages as f64 * (1.0 + Self::get_config().heap_pages_min_ratio)) as usize;
+ let target_max =
+ (used_pages as f64 * (1.0 + Self::get_config().heap_pages_max_ratio)) as usize;
+ let new_target =
+ (((used_pages as f64) * (1.0 + Self::get_config().heap_pages_goal_ratio)) as usize)
+ .clamp(
+ Self::get_config().min_heap_pages,
+ Self::get_config().max_heap_pages,
+ );
+
+ if used_pages < target_min || used_pages > target_max {
+ self.target_heap_pages.store(new_target, Ordering::Relaxed);
+ }
+ }
+ }
+
+ fn is_heap_full(&self, plan: &dyn Plan<VM = Ruby>) -> bool {
+ plan.get_reserved_pages() > self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_current_heap_size_in_pages(&self) -> usize {
+ self.target_heap_pages.load(Ordering::Relaxed)
+ }
+
+ fn get_max_heap_size_in_pages(&self) -> usize {
+ Self::get_config().max_heap_pages
+ }
+
+ fn can_heap_size_grow(&self) -> bool {
+ self.target_heap_pages.load(Ordering::Relaxed) < Self::get_config().max_heap_pages
+ }
+}
+
+impl Default for RubyHeapTrigger {
+ fn default() -> Self {
+ let min_heap_pages = Self::get_config().min_heap_pages;
+
+ Self {
+ target_heap_pages: AtomicUsize::new(min_heap_pages),
+ }
+ }
+}
+
+impl RubyHeapTrigger {
+ fn get_config<'b>() -> &'b RubyHeapTriggerConfig {
+ RUBY_HEAP_TRIGGER_CONFIG
+ .get()
+ .expect("Attempt to use RUBY_HEAP_TRIGGER_CONFIG before it is initialized")
+ }
+}
diff --git a/gc/mmtk/src/lib.rs b/gc/mmtk/src/lib.rs
index 01497e9c42..52dc782051 100644
--- a/gc/mmtk/src/lib.rs
+++ b/gc/mmtk/src/lib.rs
@@ -1,3 +1,7 @@
+// Warn about unsafe operations in functions that are already marked as unsafe.
+// This will become default in Rust 2024 edition.
+#![warn(unsafe_op_in_unsafe_fn)]
+
extern crate libc;
extern crate mmtk;
#[macro_use]
@@ -10,8 +14,11 @@ use std::sync::Mutex;
use std::thread::ThreadId;
use abi::RubyUpcalls;
-use binding::{RubyBinding, RubyBindingFast, RubyConfiguration};
-use mmtk::vm::slot::{SimpleSlot, UnimplementedMemorySlice};
+use binding::RubyBinding;
+use binding::RubyBindingFast;
+use binding::RubyConfiguration;
+use mmtk::vm::slot::SimpleSlot;
+use mmtk::vm::slot::UnimplementedMemorySlice;
use mmtk::vm::VMBinding;
use mmtk::MMTK;
use once_cell::sync::OnceCell;
@@ -21,7 +28,9 @@ pub mod active_plan;
pub mod api;
pub mod binding;
pub mod collection;
+pub mod heap;
pub mod object_model;
+pub mod pinning_registry;
pub mod reference_glue;
pub mod scanning;
pub mod utils;
@@ -51,6 +60,11 @@ impl VMBinding for Ruby {
type VMMemorySlice = RubyMemorySlice;
}
+/// The callback for mutator thread panic handler (which calls rb_bug to output
+/// debugging information such as the Ruby backtrace and memory maps).
+/// This is set before BINDING is set because mmtk_init could panic.
+pub static MUTATOR_THREAD_PANIC_HANDLER: OnceCell<extern "C" fn()> = OnceCell::new();
+
/// The singleton object for the Ruby binding itself.
pub static BINDING: OnceCell<RubyBinding> = OnceCell::new();
@@ -112,8 +126,6 @@ fn handle_gc_thread_panic(panic_info: &PanicHookInfo) {
eprintln!("Unknown backtrace status: {s:?}");
}
}
-
- std::process::abort();
}
pub(crate) fn set_panic_hook() {
@@ -126,8 +138,24 @@ pub(crate) fn set_panic_hook() {
std::panic::set_hook(Box::new(move |panic_info| {
if is_gc_thread(std::thread::current().id()) {
handle_gc_thread_panic(panic_info);
+
+ (crate::binding().upcalls().gc_thread_panic_handler)();
} else {
old_hook(panic_info);
+ (crate::MUTATOR_THREAD_PANIC_HANDLER
+ .get()
+ .expect("MUTATOR_THREAD_PANIC_HANDLER is not set"))();
}
}));
}
+
+/// This kind of assertion is enabled if either building in debug mode or the
+/// "extra_assert" feature is enabled.
+#[macro_export]
+macro_rules! extra_assert {
+ ($($arg:tt)*) => {
+ if std::cfg!(any(debug_assertions, feature = "extra_assert")) {
+ std::assert!($($arg)*);
+ }
+ };
+}
diff --git a/gc/mmtk/src/object_model.rs b/gc/mmtk/src/object_model.rs
index abeef1f2b9..d673ca11a0 100644
--- a/gc/mmtk/src/object_model.rs
+++ b/gc/mmtk/src/object_model.rs
@@ -1,8 +1,15 @@
-use crate::abi::{RubyObjectAccess, OBJREF_OFFSET};
-use crate::{abi, Ruby};
+use std::ptr::copy_nonoverlapping;
+
+use crate::abi;
+use crate::abi::RubyObjectAccess;
+use crate::abi::MIN_OBJ_ALIGN;
+use crate::abi::OBJREF_OFFSET;
+use crate::Ruby;
use mmtk::util::constants::BITS_IN_BYTE;
-use mmtk::util::copy::{CopySemantics, GCWorkerCopyContext};
-use mmtk::util::{Address, ObjectReference};
+use mmtk::util::copy::CopySemantics;
+use mmtk::util::copy::GCWorkerCopyContext;
+use mmtk::util::Address;
+use mmtk::util::ObjectReference;
use mmtk::vm::*;
pub struct VMObjectModel {}
@@ -36,13 +43,39 @@ impl ObjectModel<Ruby> for VMObjectModel {
const NEED_VO_BITS_DURING_TRACING: bool = true;
fn copy(
- _from: ObjectReference,
- _semantics: CopySemantics,
- _copy_context: &mut GCWorkerCopyContext<Ruby>,
+ from: ObjectReference,
+ semantics: CopySemantics,
+ copy_context: &mut GCWorkerCopyContext<Ruby>,
) -> ObjectReference {
- unimplemented!(
- "Copying GC not currently supported"
- )
+ let from_acc = RubyObjectAccess::from_objref(from);
+ let from_start = from_acc.obj_start();
+ let object_size = from_acc.object_size();
+ let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics);
+ debug_assert!(!to_start.is_zero());
+ let to_payload = to_start.add(OBJREF_OFFSET);
+ unsafe {
+ copy_nonoverlapping::<u8>(from_start.to_ptr(), to_start.to_mut_ptr(), object_size);
+ }
+ let to_obj = unsafe { ObjectReference::from_raw_address_unchecked(to_payload) };
+ copy_context.post_copy(to_obj, object_size, semantics);
+ trace!("Copied object from {} to {}", from, to_obj);
+
+ (crate::binding().upcalls().move_obj_during_marking)(from, to_obj);
+
+ #[cfg(feature = "clear_old_copy")]
+ {
+ trace!(
+ "Clearing old copy {} ({}-{})",
+ from,
+ from_start,
+ from_start + object_size
+ );
+ // For debug purpose, we clear the old copy so that if the Ruby VM reads from the old
+ // copy again, it will likely result in an error.
+ unsafe { std::ptr::write_bytes::<u8>(from_start.to_mut_ptr(), 0, object_size) }
+ }
+
+ to_obj
}
fn copy_to(_from: ObjectReference, _to: ObjectReference, _region: Address) -> Address {
diff --git a/gc/mmtk/src/pinning_registry.rs b/gc/mmtk/src/pinning_registry.rs
new file mode 100644
index 0000000000..b498b508f1
--- /dev/null
+++ b/gc/mmtk/src/pinning_registry.rs
@@ -0,0 +1,187 @@
+use std::sync::Mutex;
+
+use mmtk::memory_manager;
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
+use mmtk::util::ObjectReference;
+use mmtk::util::VMWorkerThread;
+use mmtk::MMTK;
+
+use crate::abi::GCThreadTLS;
+use crate::upcalls;
+use crate::Ruby;
+
+pub struct PinningRegistry {
+ pinning_objs: Mutex<Vec<ObjectReference>>,
+ pinned_objs: Mutex<Vec<ObjectReference>>,
+}
+
+impl PinningRegistry {
+ pub fn new() -> Self {
+ Self {
+ pinning_objs: Default::default(),
+ pinned_objs: Default::default(),
+ }
+ }
+
+ pub fn register(&self, object: ObjectReference) {
+ let mut pinning_objs = self.pinning_objs.lock().unwrap();
+ pinning_objs.push(object);
+ }
+
+ pub fn pin_children(&self, tls: VMWorkerThread) {
+ if !crate::mmtk().get_plan().current_gc_may_move_object() {
+ log::debug!("The current GC is non-moving, skipping pinning children.");
+ return;
+ }
+
+ let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) };
+ let worker = gc_tls.worker();
+
+ let pinning_objs = self
+ .pinning_objs
+ .try_lock()
+ .expect("PinningRegistry should not have races during GC.");
+
+ let packet_size = 512;
+ let work_packets = pinning_objs
+ .chunks(packet_size)
+ .map(|chunk| {
+ Box::new(PinPinningChildren {
+ pinning_objs: chunk.to_vec(),
+ }) as _
+ })
+ .collect();
+
+ worker.scheduler().work_buckets[WorkBucketStage::Prepare].bulk_add(work_packets);
+ }
+
+ pub fn cleanup(&self, worker: &mut GCWorker<Ruby>) {
+ worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(RemoveDeadPinnings);
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ let packet = {
+ let mut pinned_objs = self
+ .pinned_objs
+ .try_lock()
+ .expect("Unexpected contention on pinned_objs");
+ UnpinPinnedObjects {
+ objs: std::mem::take(&mut pinned_objs),
+ }
+ };
+
+ worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].add(packet);
+ } else {
+ debug!("The current GC is non-moving, skipping unpinning objects.");
+ debug_assert_eq!(
+ {
+ let pinned_objs = self
+ .pinned_objs
+ .try_lock()
+ .expect("Unexpected contention on pinned_objs");
+ pinned_objs.len()
+ },
+ 0
+ );
+ }
+ }
+}
+
+impl Default for PinningRegistry {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+struct PinPinningChildren {
+ pinning_objs: Vec<ObjectReference>,
+}
+
+impl GCWork<Ruby> for PinPinningChildren {
+ fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) {
+ let gc_tls = unsafe { GCThreadTLS::from_vwt_check(worker.tls) };
+ let mut pinned_objs = vec![];
+ let mut newly_pinned_objs = vec![];
+
+ let visit_object = |_worker, target_object: ObjectReference, pin| {
+ log::trace!(
+ " -> {} {}",
+ if pin { "(pin)" } else { " " },
+ target_object
+ );
+ if pin {
+ debug_assert!(
+ target_object.get_forwarded_object().is_none(),
+ "Trying to pin {target_object} but has been moved"
+ );
+
+ pinned_objs.push(target_object);
+ }
+ target_object
+ };
+
+ gc_tls
+ .object_closure
+ .set_temporarily_and_run_code(visit_object, || {
+ for obj in self.pinning_objs.iter().cloned() {
+ log::trace!(" Pinning: {}", obj);
+ (upcalls().call_gc_mark_children)(obj);
+ }
+ });
+
+ for target_object in pinned_objs {
+ if memory_manager::pin_object(target_object) {
+ newly_pinned_objs.push(target_object);
+ }
+ }
+
+ let mut pinned_objs = crate::binding()
+ .pinning_registry
+ .pinned_objs
+ .lock()
+ .unwrap();
+ pinned_objs.append(&mut newly_pinned_objs);
+ }
+}
+
+struct RemoveDeadPinnings;
+
+impl GCWork<Ruby> for RemoveDeadPinnings {
+ fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) {
+ log::debug!("Removing dead Pinnings...");
+
+ let registry = &crate::binding().pinning_registry;
+ {
+ let mut pinning_objs = registry
+ .pinning_objs
+ .try_lock()
+ .expect("PinningRegistry should not have races during GC.");
+
+ pinning_objs.retain_mut(|obj| {
+ if obj.is_live() {
+ let new_obj = obj.get_forwarded_object().unwrap_or(*obj);
+ *obj = new_obj;
+ true
+ } else {
+ log::trace!(" Dead Pinning removed: {}", *obj);
+ false
+ }
+ });
+ }
+ }
+}
+
+struct UnpinPinnedObjects {
+ objs: Vec<ObjectReference>,
+}
+
+impl GCWork<Ruby> for UnpinPinnedObjects {
+ fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static MMTK<Ruby>) {
+ log::debug!("Unpinning pinned objects...");
+
+ for obj in self.objs.iter() {
+ let unpinned = memory_manager::unpin_object(*obj);
+ debug_assert!(unpinned);
+ }
+ }
+}
diff --git a/gc/mmtk/src/scanning.rs b/gc/mmtk/src/scanning.rs
index 33466b9db6..355a2e7759 100644
--- a/gc/mmtk/src/scanning.rs
+++ b/gc/mmtk/src/scanning.rs
@@ -1,11 +1,20 @@
use crate::abi::GCThreadTLS;
+use crate::upcalls;
use crate::utils::ChunkedVecCollector;
-use crate::{upcalls, Ruby, RubySlot};
-use mmtk::scheduler::{GCWork, GCWorker, WorkBucketStage};
-use mmtk::util::{ObjectReference, VMWorkerThread};
-use mmtk::vm::{ObjectTracer, RootsWorkFactory, Scanning, SlotVisitor};
-use mmtk::{Mutator, MutatorContext};
+use crate::Ruby;
+use crate::RubySlot;
+use mmtk::memory_manager;
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
+use mmtk::util::ObjectReference;
+use mmtk::util::VMWorkerThread;
+use mmtk::vm::ObjectTracer;
+use mmtk::vm::RootsWorkFactory;
+use mmtk::vm::Scanning;
+use mmtk::vm::SlotVisitor;
+use mmtk::Mutator;
pub struct VMScanning {}
@@ -45,20 +54,33 @@ impl Scanning<Ruby> for VMScanning {
mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(),
"Destination is not an MMTk object. Src: {object} dst: {target_object}"
);
+
+ debug_assert!(
+ // If we are in a moving GC, all objects should be pinned by PinningRegistry.
+ // If it is requested that target_object be pinned but it is not pinned, then
+ // it is a bug because it could be moved.
+ if crate::mmtk().get_plan().current_gc_may_move_object() && pin {
+ memory_manager::is_pinned(target_object)
+ } else {
+ true
+ },
+ "Object {object} is trying to pin {target_object}"
+ );
+
let forwarded_target = object_tracer.trace_object(target_object);
if forwarded_target != target_object {
- trace!(
- " Forwarded target {} -> {}",
- target_object,
- forwarded_target
- );
+ trace!(" Forwarded target {target_object} -> {forwarded_target}");
}
forwarded_target
};
gc_tls
.object_closure
.set_temporarily_and_run_code(visit_object, || {
- (upcalls().scan_object_ruby_style)(object);
+ (upcalls().call_gc_mark_children)(object);
+
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ (upcalls().update_object_references)(object);
+ }
});
}
@@ -67,14 +89,13 @@ impl Scanning<Ruby> for VMScanning {
}
fn scan_roots_in_mutator_thread(
- tls: VMWorkerThread,
- mutator: &'static mut Mutator<Ruby>,
- mut factory: impl RootsWorkFactory<RubySlot>,
+ _tls: VMWorkerThread,
+ _mutator: &'static mut Mutator<Ruby>,
+ mut _factory: impl RootsWorkFactory<RubySlot>,
) {
- let gc_tls = unsafe { GCThreadTLS::from_vwt_check(tls) };
- Self::collect_object_roots_in("scan_thread_root", gc_tls, &mut factory, || {
- (upcalls().scan_roots_in_mutator_thread)(mutator.get_tls(), tls);
- });
+ // Do nothing. All stacks (including Ruby stacks and machine stacks) are reachable from
+ // `rb_vm_t` -> ractor -> thread -> fiber -> stacks. It is part of `ScanGCRoots` which
+ // calls `rb_gc_mark_roots` -> `rb_vm_mark`.
}
fn scan_vm_specific_roots(tls: VMWorkerThread, factory: impl RootsWorkFactory<RubySlot>) {
@@ -136,6 +157,7 @@ impl Scanning<Ruby> for VMScanning {
crate::binding()
.weak_proc
.process_weak_stuff(worker, tracer_context);
+ crate::binding().pinning_registry.cleanup(worker);
false
}
@@ -252,15 +274,15 @@ impl<F: RootsWorkFactory<RubySlot>> GCWork<Ruby> for ScanWbUnprotectedRoots<F> {
VMScanning::collect_object_roots_in("wb_unprot_roots", gc_tls, &mut self.factory, || {
for object in self.objects.iter().copied() {
if object.is_reachable() {
- debug!(
- "[wb_unprot_roots] Visiting WB-unprotected object (parent): {}",
- object
- );
- (upcalls().scan_object_ruby_style)(object);
+ debug!("[wb_unprot_roots] Visiting WB-unprotected object (parent): {object}");
+ (upcalls().call_gc_mark_children)(object);
+
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ (upcalls().update_object_references)(object);
+ }
} else {
debug!(
- "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {}",
- object
+ "[wb_unprot_roots] Skipping young WB-unprotected object (parent): {object}"
);
}
}
diff --git a/gc/mmtk/src/utils.rs b/gc/mmtk/src/utils.rs
index de929c3952..d1979eaf58 100644
--- a/gc/mmtk/src/utils.rs
+++ b/gc/mmtk/src/utils.rs
@@ -1,10 +1,13 @@
-use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
use atomic_refcell::AtomicRefCell;
-use mmtk::scheduler::{GCWork, GCWorker, WorkBucketStage};
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
-use sysinfo::System;
use crate::Ruby;
+use sysinfo::System;
pub struct ChunkedVecCollector<T> {
vecs: Vec<Vec<T>>,
@@ -97,32 +100,29 @@ pub fn default_heap_max() -> usize {
.expect("Invalid Memory size") as usize
}
-pub fn parse_capacity(input: &String, default: usize) -> usize {
+pub fn parse_capacity(input: &str) -> Option<usize> {
let trimmed = input.trim();
const KIBIBYTE: usize = 1024;
const MEBIBYTE: usize = 1024 * KIBIBYTE;
const GIBIBYTE: usize = 1024 * MEBIBYTE;
- let (val, suffix) = if let Some(pos) = trimmed.find(|c: char| !c.is_numeric()) {
- (&trimmed[..pos], &trimmed[pos..])
+ let (number, suffix) = if let Some(pos) = trimmed.find(|c: char| !c.is_numeric()) {
+ trimmed.split_at(pos)
} else {
(trimmed, "")
};
- // 1MiB is the default heap size
- match (val, suffix) {
- (number, "GiB") => number.parse::<usize>()
- .and_then(|v| Ok(v * GIBIBYTE))
- .unwrap_or(default),
- (number, "MiB") => number.parse::<usize>()
- .and_then(|v| Ok(v * MEBIBYTE))
- .unwrap_or(default),
- (number, "KiB") => number.parse::<usize>()
- .and_then(|v| Ok(v * KIBIBYTE))
- .unwrap_or(default),
- (number, suffix) if suffix.is_empty() => number.parse::<usize>().unwrap_or(default),
- (_, _) => default
+ let Ok(v) = number.parse::<usize>() else {
+ return None;
+ };
+
+ match suffix {
+ "GiB" => Some(v * GIBIBYTE),
+ "MiB" => Some(v * MEBIBYTE),
+ "KiB" => Some(v * KIBIBYTE),
+ "" => Some(v),
+ _ => None,
}
}
@@ -132,32 +132,30 @@ mod tests {
#[test]
fn test_parse_capacity_parses_bare_bytes() {
- assert_eq!(1234, parse_capacity(&String::from("1234"), 0));
+ assert_eq!(Some(1234), parse_capacity("1234"));
}
#[test]
fn test_parse_capacity_parses_kibibytes() {
- assert_eq!(10240, parse_capacity(&String::from("10KiB"), 0))
+ assert_eq!(Some(10240), parse_capacity("10KiB"));
}
#[test]
fn test_parse_capacity_parses_mebibytes() {
- assert_eq!(10485760, parse_capacity(&String::from("10MiB"), 0))
+ assert_eq!(Some(10485760), parse_capacity("10MiB"))
}
#[test]
fn test_parse_capacity_parses_gibibytes() {
- assert_eq!(10737418240, parse_capacity(&String::from("10GiB"), 0))
+ assert_eq!(Some(10737418240), parse_capacity("10GiB"))
}
#[test]
- fn test_parses_nonsense_value_as_default_max() {
- let default = 100;
-
- assert_eq!(default, parse_capacity(&String::from("notanumber"), default));
- assert_eq!(default, parse_capacity(&String::from("5tartswithanumber"), default));
- assert_eq!(default, parse_capacity(&String::from("number1nthemiddle"), default));
- assert_eq!(default, parse_capacity(&String::from("numberattheend111"), default));
- assert_eq!(default, parse_capacity(&String::from("mult1pl3numb3r5"), default));
+ fn test_parse_capacity_parses_nonsense_values() {
+ assert_eq!(None, parse_capacity("notanumber"));
+ assert_eq!(None, parse_capacity("5tartswithanumber"));
+ assert_eq!(None, parse_capacity("number1nthemiddle"));
+ assert_eq!(None, parse_capacity("numberattheend111"));
+ assert_eq!(None, parse_capacity("mult1pl3numb3r5"));
}
}
diff --git a/gc/mmtk/src/weak_proc.rs b/gc/mmtk/src/weak_proc.rs
index 11f7f5abbf..d38dbe04a4 100644
--- a/gc/mmtk/src/weak_proc.rs
+++ b/gc/mmtk/src/weak_proc.rs
@@ -1,23 +1,23 @@
use std::sync::Mutex;
-use mmtk::{
- scheduler::{GCWork, GCWorker, WorkBucketStage},
- util::ObjectReference,
- vm::ObjectTracerContext,
-};
-
-use crate::{
- abi::GCThreadTLS,
- upcalls,
- Ruby,
-};
+use mmtk::scheduler::GCWork;
+use mmtk::scheduler::GCWorker;
+use mmtk::scheduler::WorkBucketStage;
+use mmtk::util::ObjectReference;
+use mmtk::vm::ObjectTracerContext;
+
+use crate::abi::GCThreadTLS;
+use crate::upcalls;
+use crate::Ruby;
pub struct WeakProcessor {
+ non_parallel_obj_free_candidates: Mutex<Vec<ObjectReference>>,
+ parallel_obj_free_candidates: Vec<Mutex<Vec<ObjectReference>>>,
+
/// Objects that needs `obj_free` called when dying.
/// If it is a bottleneck, replace it with a lock-free data structure,
/// or add candidates in batch.
- obj_free_candidates: Mutex<Vec<ObjectReference>>,
- weak_references: Mutex<Vec<&'static mut ObjectReference>>,
+ weak_references: Mutex<Vec<ObjectReference>>,
}
impl Default for WeakProcessor {
@@ -29,47 +29,74 @@ impl Default for WeakProcessor {
impl WeakProcessor {
pub fn new() -> Self {
Self {
- obj_free_candidates: Mutex::new(Vec::new()),
+ non_parallel_obj_free_candidates: Mutex::new(Vec::new()),
+ parallel_obj_free_candidates: vec![Mutex::new(Vec::new())],
weak_references: Mutex::new(Vec::new()),
}
}
- /// Add an object as a candidate for `obj_free`.
- ///
- /// Multiple mutators can call it concurrently, so it has `&self`.
- pub fn add_obj_free_candidate(&self, object: ObjectReference) {
- let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
- obj_free_candidates.push(object);
+ pub fn init_parallel_obj_free_candidates(&mut self, num_workers: usize) {
+ debug_assert_eq!(self.parallel_obj_free_candidates.len(), 1);
+
+ for _ in 1..num_workers {
+ self.parallel_obj_free_candidates
+ .push(Mutex::new(Vec::new()));
+ }
}
- /// Add many objects as candidates for `obj_free`.
+ /// Add a batch of objects as candidates for `obj_free`.
///
- /// Multiple mutators can call it concurrently, so it has `&self`.
- pub fn add_obj_free_candidates(&self, objects: &[ObjectReference]) {
- let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
- for object in objects.iter().copied() {
- obj_free_candidates.push(object);
+ /// Amortizes mutex acquisition over the entire batch. Called when a
+ /// mutator's local buffer is flushed (buffer full or stop-the-world).
+ pub fn add_obj_free_candidates_batch(
+ &self,
+ objects: &[ObjectReference],
+ can_parallel_free: bool,
+ ) {
+ if objects.is_empty() {
+ return;
+ }
+
+ if can_parallel_free {
+ let num_buckets = self.parallel_obj_free_candidates.len();
+ for idx in 0..num_buckets {
+ let mut bucket = self.parallel_obj_free_candidates[idx].lock().unwrap();
+ for (i, &obj) in objects.iter().enumerate() {
+ if i % num_buckets == idx {
+ bucket.push(obj);
+ }
+ }
+ }
+ } else {
+ self.non_parallel_obj_free_candidates
+ .lock()
+ .unwrap()
+ .extend_from_slice(objects);
}
}
pub fn get_all_obj_free_candidates(&self) -> Vec<ObjectReference> {
- let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
- std::mem::take(obj_free_candidates.as_mut())
+ // let mut obj_free_candidates = self.obj_free_candidates.lock().unwrap();
+ let mut all_obj_free_candidates = self
+ .non_parallel_obj_free_candidates
+ .lock()
+ .unwrap()
+ .to_vec();
+
+ for candidates_mutex in &self.parallel_obj_free_candidates {
+ all_obj_free_candidates.extend(candidates_mutex.lock().unwrap().to_vec());
+ }
+
+ std::mem::take(all_obj_free_candidates.as_mut())
}
- pub fn add_weak_reference(&self, ptr: &'static mut ObjectReference) {
+ pub fn add_weak_reference(&self, object: ObjectReference) {
let mut weak_references = self.weak_references.lock().unwrap();
- weak_references.push(ptr);
+ weak_references.push(object);
}
- pub fn remove_weak_reference(&self, ptr: &ObjectReference) {
- let mut weak_references = self.weak_references.lock().unwrap();
- for (i, curr_ptr) in weak_references.iter().enumerate() {
- if *curr_ptr == ptr {
- weak_references.swap_remove(i);
- break;
- }
- }
+ pub fn weak_references_count(&self) -> usize {
+ self.weak_references.lock().unwrap().len()
}
pub fn process_weak_stuff(
@@ -77,80 +104,133 @@ impl WeakProcessor {
worker: &mut GCWorker<Ruby>,
_tracer_context: impl ObjectTracerContext<Ruby>,
) {
- worker.add_work(WorkBucketStage::VMRefClosure, ProcessObjFreeCandidates);
+ worker.add_work(
+ WorkBucketStage::VMRefClosure,
+ ProcessNonParallelObjFreeCanadidates {},
+ );
+
+ for index in 0..self.parallel_obj_free_candidates.len() {
+ worker.add_work(
+ WorkBucketStage::VMRefClosure,
+ ProcessParallelObjFreeCandidates { index },
+ );
+ }
+
worker.add_work(WorkBucketStage::VMRefClosure, ProcessWeakReferences);
worker.add_work(WorkBucketStage::Prepare, UpdateFinalizerObjIdTables);
let global_tables_count = (crate::upcalls().global_tables_count)();
let work_packets = (0..global_tables_count)
- .map(|i| {
- Box::new(UpdateGlobalTables { idx: i }) as _
- })
- .collect();
+ .map(|i| Box::new(UpdateGlobalTables { idx: i }) as _)
+ .collect();
worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(work_packets);
- worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure].bulk_add(vec![
- Box::new(UpdateWbUnprotectedObjectsList) as _,
- ]);
+ worker.scheduler().work_buckets[WorkBucketStage::VMRefClosure]
+ .bulk_add(vec![Box::new(UpdateWbUnprotectedObjectsList) as _]);
+ }
+}
+
+fn process_obj_free_candidates(obj_free_candidates: &mut Vec<ObjectReference>) {
+ // Process obj_free
+ let mut new_candidates = Vec::new();
+
+ for object in obj_free_candidates.iter().copied() {
+ if object.is_reachable() {
+ // Forward and add back to the candidate list.
+ let new_object = object.forward();
+ trace!("Forwarding obj_free candidate: {object} -> {new_object}");
+ new_candidates.push(new_object);
+ } else {
+ (upcalls().call_obj_free)(object);
+ }
+ }
+
+ *obj_free_candidates = new_candidates;
+}
+
+struct ProcessParallelObjFreeCandidates {
+ index: usize,
+}
+
+impl GCWork<Ruby> for ProcessParallelObjFreeCandidates {
+ fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
+ let mut obj_free_candidates = crate::binding().weak_proc.parallel_obj_free_candidates
+ [self.index]
+ .try_lock()
+ .expect("Lock for parallel_obj_free_candidates should not be held");
+
+ process_obj_free_candidates(&mut obj_free_candidates);
}
}
-struct ProcessObjFreeCandidates;
+struct ProcessNonParallelObjFreeCanadidates;
-impl GCWork<Ruby> for ProcessObjFreeCandidates {
+impl GCWork<Ruby> for ProcessNonParallelObjFreeCanadidates {
fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
- // If it blocks, it is a bug.
let mut obj_free_candidates = crate::binding()
.weak_proc
- .obj_free_candidates
+ .non_parallel_obj_free_candidates
.try_lock()
- .expect("It's GC time. No mutators should hold this lock at this time.");
+ .expect("Lock for non_parallel_obj_free_candidates should not be held");
- let n_cands = obj_free_candidates.len();
+ process_obj_free_candidates(&mut obj_free_candidates);
+ }
+}
- debug!("Total: {} candidates", n_cands);
+struct ProcessWeakReferences;
- // Process obj_free
- let mut new_candidates = Vec::new();
+impl GCWork<Ruby> for ProcessWeakReferences {
+ fn do_work(&mut self, worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
+ if crate::mmtk().get_plan().current_gc_may_move_object() {
+ let gc_tls: &mut GCThreadTLS = unsafe { GCThreadTLS::from_vwt_check(worker.tls) };
- for object in obj_free_candidates.iter().copied() {
- if object.is_reachable() {
- // Forward and add back to the candidate list.
- let new_object = object.forward();
- trace!(
- "Forwarding obj_free candidate: {} -> {}",
- object,
- new_object
+ let visit_object = |_worker, target_object: ObjectReference, _pin| {
+ debug_assert!(
+ mmtk::memory_manager::is_mmtk_object(target_object.to_raw_address()).is_some(),
+ "Destination is not an MMTk object"
);
- new_candidates.push(new_object);
- } else {
- (upcalls().call_obj_free)(object);
- }
- }
- *obj_free_candidates = new_candidates;
+ target_object
+ .get_forwarded_object()
+ .unwrap_or(target_object)
+ };
+
+ gc_tls
+ .object_closure
+ .set_temporarily_and_run_code(visit_object, || {
+ self.process_weak_references(true);
+ })
+ } else {
+ self.process_weak_references(false);
+ }
}
}
-struct ProcessWeakReferences;
-
-impl GCWork<Ruby> for ProcessWeakReferences {
- fn do_work(&mut self, _worker: &mut GCWorker<Ruby>, _mmtk: &'static mmtk::MMTK<Ruby>) {
+impl ProcessWeakReferences {
+ fn process_weak_references(&mut self, moving_gc: bool) {
let mut weak_references = crate::binding()
.weak_proc
.weak_references
.try_lock()
.expect("Mutators should not be holding the lock.");
- for ptr_ptr in weak_references.iter_mut() {
- if !(**ptr_ptr).is_reachable() {
- **ptr_ptr = crate::binding().weak_reference_dead_value;
- }
+ weak_references.retain_mut(|object_ptr| {
+ let object = object_ptr.get_forwarded_object().unwrap_or(*object_ptr);
+
+ if object != *object_ptr {
+ *object_ptr = object;
}
- weak_references.clear();
+ if object.is_reachable() {
+ (upcalls().handle_weak_references)(object, moving_gc);
+
+ true
+ } else {
+ false
+ }
+ });
}
}
@@ -165,11 +245,10 @@ trait GlobalTableProcessingWork {
let forward_object = |_worker, object: ObjectReference, _pin| {
debug_assert!(
mmtk::memory_manager::is_mmtk_object(object.to_raw_address()).is_some(),
- "{} is not an MMTk object",
- object
+ "{object} is not an MMTk object"
);
let result = object.forward();
- trace!("Forwarding reference: {} -> {}", object, result);
+ trace!("Forwarding reference: {object} -> {result}");
result
};
@@ -185,7 +264,6 @@ struct UpdateFinalizerObjIdTables;
impl GlobalTableProcessingWork for UpdateFinalizerObjIdTables {
fn process_table(&mut self) {
(crate::upcalls().update_finalizer_table)();
- (crate::upcalls().update_obj_id_tables)();
}
}
impl GCWork<Ruby> for UpdateFinalizerObjIdTables {
@@ -195,11 +273,14 @@ impl GCWork<Ruby> for UpdateFinalizerObjIdTables {
}
struct UpdateGlobalTables {
- idx: i32
+ idx: i32,
}
impl GlobalTableProcessingWork for UpdateGlobalTables {
fn process_table(&mut self) {
- (crate::upcalls().update_global_tables)(self.idx)
+ (crate::upcalls().update_global_tables)(
+ self.idx,
+ crate::mmtk().get_plan().current_gc_may_move_object(),
+ )
}
}
impl GCWork<Ruby> for UpdateGlobalTables {
@@ -224,14 +305,10 @@ impl GCWork<Ruby> for UpdateWbUnprotectedObjectsList {
if object.is_reachable() {
// Forward and add back to the candidate list.
let new_object = object.forward();
- trace!(
- "Forwarding WB-unprotected object: {} -> {}",
- object,
- new_object
- );
+ trace!("Forwarding WB-unprotected object: {object} -> {new_object}");
objects.insert(new_object);
} else {
- trace!("Removing WB-unprotected object from list: {}", object);
+ trace!("Removing WB-unprotected object from list: {object}");
}
}
diff --git a/gc/wbcheck/extconf.rb b/gc/wbcheck/extconf.rb
new file mode 100644
index 0000000000..18b32d820d
--- /dev/null
+++ b/gc/wbcheck/extconf.rb
@@ -0,0 +1,3 @@
+require_relative '../extconf_base'
+
+create_gc_makefile("wbcheck")
diff --git a/gc/wbcheck/wbcheck.c b/gc/wbcheck/wbcheck.c
new file mode 100644
index 0000000000..a7d4cd6ccf
--- /dev/null
+++ b/gc/wbcheck/wbcheck.c
@@ -0,0 +1,1936 @@
+#include "internal.h"
+#include "ruby/ruby.h"
+#include "ruby/assert.h"
+#include "ruby/atomic.h"
+#include "ruby/debug.h"
+#include "ruby/internal/core/rbasic.h"
+#include "ruby/st.h"
+#include "internal/object.h"
+#include "internal/array.h"
+#include "internal/class.h"
+
+#include "ruby/thread.h"
+#include "gc/gc.h"
+#include "gc/gc_impl.h"
+
+#include <stdbool.h>
+#include <stdarg.h>
+
+// Debug output control
+static bool wbcheck_debug_enabled = false;
+
+// Verification after write barrier control
+static bool wbcheck_verify_after_wb_enabled = false;
+
+// Useless write barrier warning control
+static bool wbcheck_warn_useless_wb_enabled = false;
+
+static void
+wbcheck_debug(const char *format, ...)
+{
+ if (!wbcheck_debug_enabled) return;
+
+ va_list args;
+ va_start(args, format);
+ vfprintf(stderr, format, args);
+ va_end(args);
+}
+
+#define WBCHECK_DEBUG(...) do { \
+ if (wbcheck_debug_enabled) { \
+ wbcheck_debug(__VA_ARGS__); \
+ } \
+} while (0)
+
+static void
+wbcheck_debug_obj_info_dump(VALUE obj)
+{
+ if (!wbcheck_debug_enabled) return;
+ char buff[0x100];
+ fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj));
+}
+
+// Forward declaration
+static void lock_and_maybe_gc(void *objspace_ptr);
+static void force_gc(void *objspace_ptr);
+
+// Configure wbcheck from environment variables
+static void
+wbcheck_configure_from_env(void)
+{
+ // Configure debug output based on environment variable
+ const char *debug_env = getenv("WBCHECK_DEBUG");
+ if (debug_env && (strcmp(debug_env, "1") == 0 || strcmp(debug_env, "true") == 0)) {
+ wbcheck_debug_enabled = true;
+ }
+
+ // Configure verification after write barrier based on environment variable
+ const char *verify_after_wb_env = getenv("WBCHECK_VERIFY_AFTER_WB");
+ if (verify_after_wb_env && (strcmp(verify_after_wb_env, "1") == 0 || strcmp(verify_after_wb_env, "true") == 0)) {
+ wbcheck_verify_after_wb_enabled = true;
+ }
+
+ // Configure useless write barrier warnings based on environment variable
+ const char *warn_useless_wb_env = getenv("WBCHECK_WARN_USELESS_WB");
+ if (warn_useless_wb_env && (strcmp(warn_useless_wb_env, "1") == 0 || strcmp(warn_useless_wb_env, "true") == 0)) {
+ wbcheck_warn_useless_wb_enabled = true;
+ }
+}
+
+// Define same heap sizes as the default GC
+static size_t heap_sizes[] = {
+ 32,
+ 40,
+ 48,
+ 56,
+ 64,
+ 72,
+ 80,
+ 96,
+ 128,
+ 160,
+ 256,
+ 512,
+ 640,
+ 768,
+ 1024,
+ 0
+};
+
+#define HEAP_COUNT ((int)(sizeof(heap_sizes) / sizeof(heap_sizes[0])) - 1)
+#define MAX_HEAP_SIZE (heap_sizes[(HEAP_COUNT) - 1])
+
+// Object states for verification tracking
+typedef enum {
+ WBCHECK_STATE_CLEAR, // Just allocated or writebarrier_remember, needs reference capture
+ WBCHECK_STATE_MARKED, // Has valid snapshot, ready for normal operation
+ WBCHECK_STATE_DIRTY // Has seen writebarrier since last snapshot, queued for verification
+} wbcheck_object_state_t;
+
+// Tri-color marking colors
+typedef enum {
+ WBCHECK_COLOR_WHITE, // Unmarked - will be swept
+ WBCHECK_COLOR_GRAY, // Marked but children not processed
+ WBCHECK_COLOR_BLACK // Marked and children processed
+} wbcheck_color_t;
+
+// GC phases
+typedef enum {
+ WBCHECK_PHASE_MUTATOR, // Normal execution
+ WBCHECK_PHASE_SNAPSHOT, // Collecting references for verification
+ WBCHECK_PHASE_FULL_GC // Marking objects during full GC
+} wbcheck_phase_t;
+
+// List of objects
+typedef struct {
+ VALUE *items;
+ size_t count;
+ size_t capacity;
+} wbcheck_object_list_t;
+
+// Helper functions for object list
+static wbcheck_object_list_t *
+wbcheck_object_list_init_with_capacity(size_t capacity)
+{
+ wbcheck_object_list_t *list = calloc(1, sizeof(wbcheck_object_list_t));
+ if (!list) rb_bug("wbcheck: failed to allocate object list structure");
+
+ if (capacity < 4) capacity = 4;
+ list->items = malloc(capacity * sizeof(VALUE));
+ if (!list->items) rb_bug("wbcheck: failed to allocate object list array");
+ list->capacity = capacity;
+ list->count = 0;
+ return list;
+}
+
+static wbcheck_object_list_t *
+wbcheck_object_list_init(void)
+{
+ return wbcheck_object_list_init_with_capacity(4);
+}
+
+static void
+wbcheck_object_list_append(wbcheck_object_list_t *list, VALUE obj)
+{
+ if (list->count >= list->capacity) {
+ size_t new_capacity = list->capacity == 0 ? 4 : list->capacity * 2;
+ VALUE *new_items = realloc(list->items, new_capacity * sizeof(VALUE));
+ if (!new_items) rb_bug("wbcheck: failed to reallocate object list array");
+ list->items = new_items;
+ list->capacity = new_capacity;
+ }
+ list->items[list->count++] = obj;
+}
+
+static void
+wbcheck_object_list_free(wbcheck_object_list_t *list)
+{
+ if (!list) return;
+ if (list->items) {
+ free(list->items);
+ }
+ free(list);
+}
+
+static void
+wbcheck_object_list_debug_print(wbcheck_object_list_t *list)
+{
+ if (!wbcheck_debug_enabled) return;
+ for (size_t i = 0; i < list->count; i++) {
+ char buff[0x100];
+ fprintf(stderr, "-> %s\n", rb_raw_obj_info(buff, sizeof(buff), list->items[i]));
+ }
+}
+
+static bool
+wbcheck_object_list_contains(wbcheck_object_list_t *list, VALUE obj)
+{
+ for (size_t i = 0; i < list->count; i++) {
+ if (list->items[i] == obj) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// Information tracked for each object
+typedef struct {
+ size_t alloc_size; // Allocated size (static)
+ bool wb_protected; // Write barrier protection status (static)
+ VALUE finalizers; // Ruby Array of finalizers like [finalizer1, finalizer2, ...]
+ wbcheck_object_list_t *gc_mark_snapshot; // Snapshot of references from last GC mark
+ wbcheck_object_list_t *mark_maybe_snapshot; // Conservative refs reported via mark_maybe; needed for liveness, not verifiable
+ wbcheck_object_list_t *writebarrier_children; // References added via write barriers since last snapshot
+ wbcheck_object_state_t state; // Current state in verification lifecycle
+ wbcheck_color_t color; // Tri-color marking color
+} rb_wbcheck_object_info_t;
+
+// Finalizer job types
+struct wbcheck_final_job {
+ struct wbcheck_final_job *next;
+ enum {
+ WBCHECK_FINAL_JOB_DFREE,
+ WBCHECK_FINAL_JOB_FINALIZE,
+ } kind;
+ union {
+ struct {
+ void (*func)(void *);
+ void *data;
+ } dfree;
+ struct {
+ VALUE finalizer_array;
+ } finalize;
+ } as;
+};
+
+// wbcheck objspace structure to track all objects
+typedef struct {
+ st_table *object_table; // Hash table to track all allocated objects (VALUE -> rb_wbcheck_object_info_t*)
+ wbcheck_object_list_t *objects_to_capture; // Objects that need initial reference capture
+ wbcheck_object_list_t *objects_to_verify; // Objects that need verification after write barriers
+ wbcheck_object_list_t *current_refs; // Current list for collecting references during marking
+ wbcheck_object_list_t *current_maybe_refs; // Current list for collecting mark_maybe references during marking
+ wbcheck_object_list_t *mark_queue; // Queue of gray objects for tri-color marking
+ wbcheck_object_list_t *weak_references; // Objects holding weak references, found during marking
+ wbcheck_phase_t phase; // Current GC phase
+ bool gc_enabled; // Whether GC is allowed to run
+ bool gc_stress; // GC stress mode (run GC on every allocation)
+ size_t gc_threshold; // Trigger GC when object count reaches this
+ size_t missed_write_barrier_parents; // Number of parent objects with missed write barriers
+ size_t missed_write_barrier_children; // Total number of missed write barriers detected
+ size_t simulated_gc_count; // Simulated GC count incremented on each GC.start
+ bool measure_total_time; // Whether to accumulate :time in stats
+ struct wbcheck_final_job *finalizer_jobs; // Linked list of finalizer jobs
+ rb_nativethread_lock_t finalizer_lock; // Protects finalizer_jobs list
+ rb_postponed_job_handle_t finalizer_postponed_job; // Postponed job handle for finalizers
+} rb_wbcheck_objspace_t;
+
+// Global objspace pointer for accessing from obj_slot_size function
+static rb_wbcheck_objspace_t *wbcheck_global_objspace = NULL;
+
+// Forward declarations
+static void wbcheck_foreach_object(rb_wbcheck_objspace_t *objspace, int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data), void *data);
+static int wbcheck_verify_all_references_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data);
+static int wbcheck_update_all_snapshots_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data);
+static void wbcheck_run_finalizers_for_object(VALUE obj, rb_wbcheck_object_info_t *info);
+static void gc_run_finalizers(void *data);
+static void make_final_job(rb_wbcheck_objspace_t *objspace, VALUE obj, VALUE finalizer_array);
+
+// Helper functions for object tracking
+static rb_wbcheck_object_info_t *
+wbcheck_get_object_info(VALUE obj)
+{
+ // Objspace must be initialized by this point
+ GC_ASSERT(wbcheck_global_objspace);
+
+ st_data_t value;
+ if (st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value)) {
+ return (rb_wbcheck_object_info_t *)value;
+ }
+
+ fprintf(stderr, "wbcheck: object not found in tracking table\n");
+ char buff[0x100];
+ fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj));
+
+ // Force ASAN crash?
+ ((volatile VALUE *)obj)[0];
+
+ // Object not found in tracking table - this should never happen
+ rb_bug("wbcheck: object not found in tracking table");
+}
+
+static void
+wbcheck_report_error(void *objspace_ptr, VALUE parent_obj, wbcheck_object_list_t *current_refs, wbcheck_object_list_t *gc_mark_snapshot, wbcheck_object_list_t *writebarrier_children, wbcheck_object_list_t *missed_refs)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+
+ rb_wbcheck_object_info_t *parent_info = wbcheck_get_object_info(parent_obj);
+
+ size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0;
+ size_t wb_count = writebarrier_children ? writebarrier_children->count : 0;
+
+ fprintf(stderr, "WBCHECK ERROR: Missed write barrier detected!\n");
+ fprintf(stderr, " Parent object: %p (wb_protected: %s)\n",
+ (void *)parent_obj, parent_info->wb_protected ? "true" : "false");
+ char buff[0x100];
+ fprintf(stderr, " %s\n", rb_raw_obj_info(buff, sizeof(buff), parent_obj));
+ fprintf(stderr, " Reference counts - snapshot: %zu, writebarrier: %zu, current: %zu, missed: %zu\n",
+ snapshot_count, wb_count, current_refs->count, missed_refs->count);
+
+ for (size_t i = 0; i < missed_refs->count; i++) {
+ VALUE missed_ref = missed_refs->items[i];
+ char buff[0x100];
+ fprintf(stderr, " Missing reference to: %p\n %s\n", (void *)missed_ref, rb_raw_obj_info(buff, sizeof(buff), missed_ref));
+ }
+
+ fprintf(stderr, "\n");
+ objspace->missed_write_barrier_parents++;
+ objspace->missed_write_barrier_children += missed_refs->count;
+}
+
+static void
+wbcheck_compare_references(void *objspace_ptr, VALUE parent_obj, wbcheck_object_list_t *current_refs, wbcheck_object_list_t *gc_mark_snapshot, wbcheck_object_list_t *writebarrier_children)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ (void)objspace;
+
+ size_t snapshot_count = gc_mark_snapshot ? gc_mark_snapshot->count : 0;
+ size_t wb_count = writebarrier_children ? writebarrier_children->count : 0;
+
+ WBCHECK_DEBUG("wbcheck: comparing references for object %p\n", (void *)parent_obj);
+ WBCHECK_DEBUG("wbcheck: current refs: %zu, snapshot refs: %zu, wb refs: %zu\n",
+ current_refs->count, snapshot_count, wb_count);
+
+ // Collect missed references (lazily allocated)
+ wbcheck_object_list_t *missed_refs = NULL;
+
+ // Use circular comparison for better performance when lists are mostly similar
+ size_t snapshot_idx = 0;
+
+ // Check each object in current_refs to see if it's in either stored list
+ for (size_t i = 0; i < current_refs->count; i++) {
+ VALUE current_ref = current_refs->items[i];
+
+ // Usually the lists are nearly identical. We take advantage of this by
+ // attempting to loop over both lists in sequence. When the next element
+ // of the snapshot doesn't match the next element of our current_refs,
+ // we'll loop around the list to try to find it and continue from that
+ // match, so any runs of identical items can be matched efficiently.
+ //
+ // Pathologically this is O(N**2), but is O(N * num_changes)
+ bool found_in_snapshot = false;
+ if (gc_mark_snapshot && snapshot_count > 0) {
+ size_t start_idx = snapshot_idx;
+ do {
+ if (gc_mark_snapshot->items[snapshot_idx] == current_ref) {
+ found_in_snapshot = true;
+ snapshot_idx++;
+ if (snapshot_idx >= snapshot_count) snapshot_idx = 0;
+ break;
+ }
+ snapshot_idx++;
+ if (snapshot_idx >= snapshot_count) snapshot_idx = 0;
+ } while (snapshot_idx != start_idx);
+ }
+
+ if (found_in_snapshot) {
+ continue;
+ }
+
+ // Built-in immortal classes can be assigned via RBASIC_SET_CLASS_RAW,
+ // which bypasses the write barrier. They're pinned as VM roots and
+ // can never be collected, so a missing WB to them is harmless.
+ if (RB_TYPE_P(current_ref, T_CLASS) && FL_TEST_RAW(current_ref, RCLASS_IS_ROOT)) {
+ continue;
+ }
+
+ // Self reference... Weird but okay I guess
+ if (current_ref == parent_obj) {
+ continue;
+ }
+
+
+ // Check if reference exists in writebarrier_children
+ if (writebarrier_children && wbcheck_object_list_contains(writebarrier_children, current_ref)) {
+ continue;
+ }
+
+ // If we get here, the reference wasn't found in either list
+ // Lazily allocate missed_refs list on first miss
+ if (!missed_refs) {
+ missed_refs = wbcheck_object_list_init();
+ }
+ wbcheck_object_list_append(missed_refs, current_ref);
+ }
+
+ // Report any errors found
+ if (missed_refs) {
+ wbcheck_report_error(objspace_ptr, parent_obj, current_refs, gc_mark_snapshot, writebarrier_children, missed_refs);
+ wbcheck_object_list_free(missed_refs);
+ }
+}
+
+static void
+wbcheck_register_object(void *objspace_ptr, VALUE obj, size_t alloc_size, bool wb_protected)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ GC_ASSERT(objspace);
+
+ // Allocate and initialize object info structure
+ rb_wbcheck_object_info_t *info = calloc(1, sizeof(rb_wbcheck_object_info_t));
+ if (!info) rb_bug("wbcheck_register_object: failed to allocate object info");
+
+ info->alloc_size = alloc_size;
+ info->wb_protected = wb_protected;
+ info->finalizers = 0; /* No finalizers initially */
+ info->gc_mark_snapshot = NULL; /* No snapshot initially */
+ info->mark_maybe_snapshot = NULL; /* No mark_maybe snapshot initially */
+ info->writebarrier_children = NULL; /* No write barrier children initially */
+ info->state = WBCHECK_STATE_CLEAR; /* Start in clear state */
+ info->color = WBCHECK_COLOR_BLACK; /* Start as black to survive current GC */
+
+ // Store object info in hash table (VALUE -> rb_wbcheck_object_info_t*)
+ st_insert(objspace->object_table, (st_data_t)obj, (st_data_t)info);
+}
+
+static void
+wbcheck_unregister_object(void *objspace_ptr, VALUE obj)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ rb_wbcheck_object_info_t *info;
+
+ if (st_delete(objspace->object_table, (st_data_t *)&obj, (st_data_t *)&info)) {
+ // Free object lists if they were allocated
+ wbcheck_object_list_free(info->gc_mark_snapshot);
+ wbcheck_object_list_free(info->mark_maybe_snapshot);
+ wbcheck_object_list_free(info->writebarrier_children);
+ free(info);
+ } else {
+ rb_bug("wbcheck_unregister_object: object not found in table");
+ }
+}
+
+// Bootup
+void *
+rb_gc_impl_objspace_alloc(void)
+{
+ wbcheck_configure_from_env();
+
+ rb_wbcheck_objspace_t *objspace = calloc(1, sizeof(rb_wbcheck_objspace_t));
+ if (!objspace) rb_bug("wbcheck: failed to allocate objspace");
+
+ objspace->object_table = st_init_numtable();
+ if (!objspace->object_table) {
+ free(objspace);
+ rb_bug("wbcheck: failed to create object table");
+ }
+
+ objspace->objects_to_capture = wbcheck_object_list_init(); // Initialize empty list
+ objspace->objects_to_verify = wbcheck_object_list_init(); // Initialize empty list
+ objspace->current_refs = NULL; // No current refs initially
+ objspace->current_maybe_refs = NULL; // No current maybe refs initially
+ objspace->mark_queue = wbcheck_object_list_init(); // Initialize mark queue
+ objspace->weak_references = wbcheck_object_list_init(); // Initialize weak references array
+ objspace->phase = WBCHECK_PHASE_MUTATOR; // Start in mutator phase
+ objspace->gc_enabled = true; // GC enabled by default (like default GC)
+ objspace->gc_stress = false; // GC stress disabled by default
+ objspace->gc_threshold = 1000; // Start with 1000 objects, will adjust after first GC
+ objspace->missed_write_barrier_parents = 0; // No errors found yet
+ objspace->missed_write_barrier_children = 0; // No errors found yet
+ objspace->simulated_gc_count = 0; // Start with GC count of 0
+ objspace->measure_total_time = true; // On by default
+
+ return objspace;
+}
+
+void
+rb_gc_impl_objspace_init(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = objspace_ptr;
+
+ // Object table is already initialized in objspace_alloc
+ // Set up global objspace pointer for obj_slot_size function
+ wbcheck_global_objspace = objspace;
+
+ // Initialize postponed job for finalizers
+ rb_native_mutex_initialize(&objspace->finalizer_lock);
+ objspace->finalizer_postponed_job = rb_postponed_job_preregister(0, gc_run_finalizers, objspace);
+}
+
+void *
+rb_gc_impl_ractor_cache_alloc(void *objspace_ptr, void *ractor)
+{
+ // Stub implementation
+ return NULL;
+}
+
+void
+rb_gc_impl_set_params(void *objspace_ptr)
+{
+ // Stub implementation
+}
+
+static VALUE
+gc_verify_internal_consistency(VALUE self)
+{
+ return Qnil;
+}
+
+void
+rb_gc_impl_init(void)
+{
+ VALUE gc_constants = rb_hash_new();
+ //rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX])));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MAX_HEAP_SIZE));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(HEAP_COUNT));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(HEAP_COUNT));
+ rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(3));
+ OBJ_FREEZE(gc_constants);
+ rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
+
+ // no-ops for compatibility
+ rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0);
+
+ rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0);
+ rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0);
+ rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1);
+ rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
+ rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
+ // Stub implementation
+}
+
+size_t *
+rb_gc_impl_heap_sizes(void *objspace_ptr)
+{
+ return heap_sizes;
+}
+
+// Shutdown
+void
+rb_gc_impl_shutdown_free_objects(void *objspace_ptr)
+{
+ // Stub implementation
+}
+
+void
+rb_gc_impl_objspace_free(void *objspace_ptr)
+{
+ // This should free everything, but we'll just let it leak
+}
+
+void
+rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache)
+{
+ // Stub implementation
+}
+
+// GC
+void
+rb_gc_impl_start(void *objspace_ptr, bool full_mark, bool immediate_mark, bool immediate_sweep, bool compact)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ if (objspace) {
+ objspace->simulated_gc_count++;
+ }
+
+ if (!ruby_native_thread_p()) return;
+
+ unsigned int lev = RB_GC_VM_LOCK();
+ rb_gc_vm_barrier();
+ force_gc(objspace_ptr);
+ RB_GC_VM_UNLOCK(lev);
+}
+
+bool
+rb_gc_impl_during_gc_p(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ return objspace->phase != WBCHECK_PHASE_MUTATOR;
+}
+
+static void
+wbcheck_prepare_heap_i(VALUE obj, void *data)
+{
+ rb_gc_prepare_heap_process_object(obj);
+}
+
+void
+rb_gc_impl_prepare_heap(void *objspace_ptr)
+{
+ rb_gc_impl_each_object(objspace_ptr, wbcheck_prepare_heap_i, NULL);
+}
+
+void
+rb_gc_impl_gc_enable(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ objspace->gc_enabled = true;
+}
+
+void
+rb_gc_impl_gc_disable(void *objspace_ptr, bool finish_current_gc)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ objspace->gc_enabled = false;
+}
+
+bool
+rb_gc_impl_gc_enabled_p(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ return objspace->gc_enabled;
+}
+
+void
+rb_gc_impl_stress_set(void *objspace_ptr, VALUE flag)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ objspace->gc_stress = RTEST(flag);
+}
+
+VALUE
+rb_gc_impl_stress_get(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ return objspace->gc_stress ? Qtrue : Qfalse;
+}
+
+VALUE
+rb_gc_impl_config_get(void *objspace_ptr)
+{
+ return rb_hash_new();
+}
+
+void
+rb_gc_impl_config_set(void *objspace_ptr, VALUE hash)
+{
+}
+
+static wbcheck_object_list_t *
+wbcheck_collect_references_from_object(VALUE obj, rb_wbcheck_object_info_t *info)
+{
+ rb_wbcheck_objspace_t *objspace = wbcheck_global_objspace;
+
+ // Use combination of writebarrier children and last snapshot as capacity hint
+ size_t snapshot_count = (info->gc_mark_snapshot) ? info->gc_mark_snapshot->count : 0;
+ size_t wb_children_count = (info->writebarrier_children) ? info->writebarrier_children->count : 0;
+ size_t capacity_hint = snapshot_count + wb_children_count;
+ wbcheck_object_list_t *new_list = wbcheck_object_list_init_with_capacity(capacity_hint);
+
+ // Set up objspace state for marking. current_maybe_refs is allocated lazily
+ // by rb_gc_impl_mark_maybe, since most objects have no conservative refs.
+ objspace->current_refs = new_list;
+ objspace->current_maybe_refs = NULL;
+ objspace->phase = WBCHECK_PHASE_SNAPSHOT;
+
+ // Use the marking infrastructure to collect references
+ rb_gc_mark_children(objspace, obj);
+
+ // Clean up objspace state
+ objspace->phase = WBCHECK_PHASE_MUTATOR;
+ objspace->current_refs = NULL;
+
+ // Update the mark_maybe snapshot in place. These references don't participate
+ // in verification, but we need to keep them so full GC can mark them gray.
+ wbcheck_object_list_free(info->mark_maybe_snapshot);
+ info->mark_maybe_snapshot = objspace->current_maybe_refs;
+ objspace->current_maybe_refs = NULL;
+
+ if (wbcheck_debug_enabled) {
+ WBCHECK_DEBUG("wbcheck: collected %zu references from %p\n", new_list->count, (void *)obj);
+ char buff[0x100];
+ fprintf(stderr, "%s\n", rb_raw_obj_info(buff, sizeof(buff), obj));
+ wbcheck_object_list_debug_print(new_list);
+ }
+
+ return new_list;
+}
+
+static void
+wbcheck_collect_initial_references(void *objspace_ptr, VALUE obj)
+{
+ WBCHECK_DEBUG("wbcheck: collecting initial references from %p:\n", obj);
+ wbcheck_debug_obj_info_dump(obj);
+
+ // Get the object info and set the initial GC mark snapshot
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj);
+ wbcheck_object_list_t *new_list = wbcheck_collect_references_from_object(obj, info);
+ RUBY_ASSERT(!info->gc_mark_snapshot);
+ RUBY_ASSERT(info->state == WBCHECK_STATE_CLEAR);
+ info->gc_mark_snapshot = new_list; // Set the initial snapshot
+ info->state = WBCHECK_STATE_MARKED; // Transition to marked state
+}
+
+static void
+wbcheck_verify_object_references(void *objspace_ptr, VALUE obj)
+{
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj);
+
+ // Ignore objects which are not write barrier protected
+ if (!info->wb_protected) {
+ return;
+ }
+
+ // We hadn't captured initial references
+ if (info->state == WBCHECK_STATE_CLEAR) {
+ RUBY_ASSERT(!info->gc_mark_snapshot);
+ return;
+ }
+
+ WBCHECK_DEBUG("wbcheck: verifying references for object:\n");
+ wbcheck_debug_obj_info_dump(obj);
+
+ // Get the current references from the object
+ wbcheck_object_list_t *current_refs = wbcheck_collect_references_from_object(obj, info);
+
+ // Check for useless write barriers before clearing them
+ if (wbcheck_warn_useless_wb_enabled && info->writebarrier_children) {
+ for (size_t i = 0; i < info->writebarrier_children->count; i++) {
+ VALUE wb_ref = info->writebarrier_children->items[i];
+ if (!wbcheck_object_list_contains(current_refs, wb_ref)) {
+ fprintf(stderr, "WBCHECK WARNING: Potentially useless write barrier detected for object %p\n", (void *)obj);
+ fprintf(stderr, " Write barrier was recorded for reference to %p, but object no longer references it\n", (void *)wb_ref);
+ char buff[0x100];
+ fprintf(stderr, " Parent: %s\n", rb_raw_obj_info(buff, sizeof(buff), obj));
+ fprintf(stderr, " Stale reference: %s\n", rb_raw_obj_info(buff, sizeof(buff), wb_ref));
+ }
+ }
+ }
+
+ // Compare current_refs against both stored lists to detect missed write barriers
+ wbcheck_compare_references(objspace_ptr, obj, current_refs, info->gc_mark_snapshot, info->writebarrier_children);
+
+ // Update the snapshot with current references and clear write barrier children
+ wbcheck_object_list_free(info->gc_mark_snapshot);
+ wbcheck_object_list_free(info->writebarrier_children);
+ info->gc_mark_snapshot = current_refs;
+ info->writebarrier_children = NULL;
+ info->state = WBCHECK_STATE_MARKED; // Back to marked state after verification
+}
+
+// Mark object as gray (add to mark queue)
+static void
+wbcheck_mark_gray(rb_wbcheck_objspace_t *objspace, VALUE obj)
+{
+ if (RB_SPECIAL_CONST_P(obj)) return;
+
+ st_data_t value;
+ if (!st_lookup(objspace->object_table, (st_data_t)obj, &value)) {
+ rb_bug("wbcheck: asked to mark object %p not in our object table", (void *)obj);
+ }
+
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value;
+ if (info->color != WBCHECK_COLOR_WHITE) {
+ return; // Already marked
+ }
+
+ info->color = WBCHECK_COLOR_GRAY;
+ wbcheck_object_list_append(objspace->mark_queue, obj);
+
+ if (RB_FL_TEST_RAW(obj, RUBY_FL_WEAK_REFERENCE)) {
+ wbcheck_object_list_append(objspace->weak_references, obj);
+ }
+
+ WBCHECK_DEBUG("wbcheck: marked gray: %p\n", (void *)obj);
+}
+
+// Reset all objects to white
+static int
+st_foreach_reset_white(st_data_t key, st_data_t val, st_data_t arg)
+{
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val;
+ info->color = WBCHECK_COLOR_WHITE;
+ return ST_CONTINUE;
+}
+
+// Mark all finalizer arrays to keep them alive during GC
+static int
+st_foreach_mark_finalizers(st_data_t key, st_data_t val, st_data_t arg)
+{
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val;
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)arg;
+
+ if (info->finalizers) {
+ wbcheck_mark_gray(objspace, info->finalizers);
+ }
+
+ return ST_CONTINUE;
+}
+
+// Full mark phase using tri-color marking with snapshots
+static void
+wbcheck_mark_phase(rb_wbcheck_objspace_t *objspace)
+{
+ WBCHECK_DEBUG("wbcheck: starting GC mark phase\n");
+
+ objspace->phase = WBCHECK_PHASE_FULL_GC;
+
+ // Clear mark queue and reset all objects to white
+ objspace->mark_queue->count = 0;
+ st_foreach(objspace->object_table, st_foreach_reset_white, 0);
+
+ // Mark all finalizer arrays first to keep them alive
+ st_foreach(objspace->object_table, st_foreach_mark_finalizers, (st_data_t)objspace);
+
+ // Mark finalizer arrays in pending jobs to keep them alive.
+ // No lock needed: all other threads are stopped during GC.
+ struct wbcheck_final_job *job = objspace->finalizer_jobs;
+ while (job != NULL) {
+ switch (job->kind) {
+ case WBCHECK_FINAL_JOB_DFREE:
+ break;
+ case WBCHECK_FINAL_JOB_FINALIZE:
+ wbcheck_mark_gray(objspace, job->as.finalize.finalizer_array);
+ break;
+ default:
+ rb_bug("wbcheck_mark_phase: unknown final job type %d", job->kind);
+ }
+ job = job->next;
+ }
+
+ // Mark roots gray
+ rb_gc_save_machine_context();
+ rb_gc_mark_roots(objspace, NULL);
+
+ // Process gray queue until empty
+ while (objspace->mark_queue->count > 0) {
+ // Get last object from queue (LIFO)
+ VALUE obj = objspace->mark_queue->items[--objspace->mark_queue->count];
+
+ st_data_t value;
+ if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) {
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value;
+ if (info->color == WBCHECK_COLOR_GRAY) {
+ // Mark all children from snapshot gray
+ if (info->gc_mark_snapshot) {
+ for (size_t i = 0; i < info->gc_mark_snapshot->count; i++) {
+ wbcheck_mark_gray(objspace, info->gc_mark_snapshot->items[i]);
+ }
+ }
+
+ // Conservatively-scanned children must also be kept alive
+ if (info->mark_maybe_snapshot) {
+ for (size_t i = 0; i < info->mark_maybe_snapshot->count; i++) {
+ wbcheck_mark_gray(objspace, info->mark_maybe_snapshot->items[i]);
+ }
+ }
+
+ // Mark this object black
+ info->color = WBCHECK_COLOR_BLACK;
+ WBCHECK_DEBUG("wbcheck: marked black: %p\n", (void *)obj);
+ }
+ }
+ }
+
+ objspace->phase = WBCHECK_PHASE_MUTATOR;
+
+ WBCHECK_DEBUG("wbcheck: tri-color mark phase complete\n");
+}
+
+// Sweep phase callback - free white objects
+static int
+wbcheck_sweep_callback(st_data_t key, st_data_t val, st_data_t arg, int error)
+{
+ VALUE obj = (VALUE)key;
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val;
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)arg;
+
+ if (info->color == WBCHECK_COLOR_WHITE) {
+ WBCHECK_DEBUG("wbcheck: sweeping unmarked object %p\n", (void *)obj);
+
+ rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_FREEOBJ);
+
+ // Clear weak references first
+ rb_gc_obj_free_vm_weak_references(obj);
+
+ // Queue finalizers for postponed job if they exist
+ if (info->finalizers) {
+ make_final_job(objspace, obj, info->finalizers);
+ rb_postponed_job_trigger(objspace->finalizer_postponed_job);
+ }
+
+ // Call rb_gc_obj_free which handles finalizers/zombies
+ if (rb_gc_obj_free(objspace, obj)) {
+ // Object was actually freed, clean up our tracking
+ wbcheck_object_list_free(info->gc_mark_snapshot);
+ wbcheck_object_list_free(info->mark_maybe_snapshot);
+ wbcheck_object_list_free(info->writebarrier_children);
+ free(info);
+
+ // Free the actual object memory
+ free((void *)obj);
+
+ return ST_DELETE; // Remove from hash table
+ } else {
+ // Object became a zombie - it will be freed by postponed job
+ // Remove from tracking since we can't safely access it anymore
+ wbcheck_object_list_free(info->gc_mark_snapshot);
+ wbcheck_object_list_free(info->mark_maybe_snapshot);
+ wbcheck_object_list_free(info->writebarrier_children);
+ free(info);
+
+ // Free the actual object memory
+ free((void *)obj);
+
+ return ST_DELETE; // Remove from hash table
+ }
+ }
+
+ return ST_CONTINUE; // Keep marked objects
+}
+
+static void
+wbcheck_sweep_phase(rb_wbcheck_objspace_t *objspace)
+{
+ WBCHECK_DEBUG("wbcheck: starting sweep phase\n");
+
+ size_t objects_before = st_table_size(objspace->object_table);
+
+ // Sweep unmarked objects
+ st_foreach_check(objspace->object_table, wbcheck_sweep_callback, (st_data_t)objspace, 0);
+
+ size_t objects_after = st_table_size(objspace->object_table);
+ size_t freed_objects = objects_before - objects_after;
+
+ // Update GC threshold: 2x the live set after GC
+ objspace->gc_threshold = objects_after * 2;
+
+ WBCHECK_DEBUG("wbcheck: sweep phase complete - freed %zu objects (%zu -> %zu), new threshold: %zu\n",
+ freed_objects, objects_before, objects_after, objspace->gc_threshold);
+}
+
+// Process weak references after marking - call rb_gc_handle_weak_references
+// on each object that was flagged with RUBY_FL_WEAK_REFERENCE and collected
+// during the mark phase.
+static void
+wbcheck_process_weak_references(rb_wbcheck_objspace_t *objspace)
+{
+ WBCHECK_DEBUG("wbcheck: processing %zu weak reference objects\n", objspace->weak_references->count);
+
+ for (size_t i = 0; i < objspace->weak_references->count; i++) {
+ VALUE obj = objspace->weak_references->items[i];
+ rb_gc_handle_weak_references(obj);
+ }
+
+ objspace->weak_references->count = 0;
+}
+
+// Full GC: verify all objects then mark from roots
+static void
+wbcheck_full_gc(rb_wbcheck_objspace_t *objspace)
+{
+ WBCHECK_DEBUG("wbcheck: starting full GC\n");
+
+ rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_ENTER);
+ rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_START);
+
+ // First, update snapshots for all objects (verify wb_protected ones)
+ WBCHECK_DEBUG("wbcheck: updating snapshots for all objects\n");
+ wbcheck_foreach_object(objspace, wbcheck_update_all_snapshots_callback, objspace);
+
+ // Now start tri-color marking
+ wbcheck_mark_phase(objspace);
+
+ rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_MARK);
+
+ // Process weak references after marking, before sweeping
+ wbcheck_process_weak_references(objspace);
+
+ // Sweep unmarked objects
+ wbcheck_sweep_phase(objspace);
+
+ rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_END_SWEEP);
+ rb_gc_event_hook(0, RUBY_INTERNAL_EVENT_GC_EXIT);
+
+ WBCHECK_DEBUG("wbcheck: full GC complete\n");
+}
+
+static void
+gc_step(void *objspace_ptr, bool force)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+
+ // Not initialized yet
+ if (!objspace) return;
+
+ if (!objspace->gc_enabled && !force) return;
+
+ // Process all objects that need verification after write barriers (if enabled)
+ if (wbcheck_verify_after_wb_enabled) {
+ for (size_t i = 0; i < objspace->objects_to_verify->count; i++) {
+ VALUE obj = objspace->objects_to_verify->items[i];
+ wbcheck_verify_object_references(objspace_ptr, obj);
+ }
+
+ // Clear the list after processing
+ objspace->objects_to_verify->count = 0;
+
+ // If any new errors were detected during verification, exit immediately
+ if (objspace->missed_write_barrier_parents > 0) {
+ rb_bug("wbcheck: missed write barrier detected during immediate verification (WBCHECK_VERIFY_AFTER_WB=1)");
+ }
+ }
+
+ // Process all objects that need initial reference capture
+ for (size_t i = 0; i < objspace->objects_to_capture->count; i++) {
+ VALUE obj = objspace->objects_to_capture->items[i];
+ wbcheck_collect_initial_references(objspace_ptr, obj);
+ }
+
+ // Clear the list after processing
+ objspace->objects_to_capture->count = 0;
+
+ // Run full GC if forced, if we exceed the threshold, or if gc_stress is enabled
+ if (ruby_native_thread_p() &&
+ (force ||
+ (objspace->gc_enabled &&
+ (objspace->gc_stress || st_table_size(objspace->object_table) >= objspace->gc_threshold)))) {
+ wbcheck_full_gc(objspace);
+ }
+
+}
+
+static void
+maybe_gc(void *objspace_ptr)
+{
+ gc_step(objspace_ptr, false);
+}
+
+static void
+force_gc(void *objspace_ptr)
+{
+ gc_step(objspace_ptr, true);
+}
+
+int ruby_thread_has_gvl_p(void);
+
+static void *
+lock_and_maybe_gc_gvl(void *objspace_ptr)
+{
+ unsigned int lev = RB_GC_VM_LOCK();
+ rb_gc_vm_barrier();
+
+ maybe_gc(objspace_ptr);
+
+ RB_GC_VM_UNLOCK(lev);
+ return NULL;
+}
+
+static void
+lock_and_maybe_gc(void *objspace_ptr)
+{
+ if (!ruby_native_thread_p()) return;
+
+ if (!ruby_thread_has_gvl_p()) {
+ rb_thread_call_with_gvl(lock_and_maybe_gc_gvl, objspace_ptr);
+ }
+ else {
+ lock_and_maybe_gc_gvl(objspace_ptr);
+ }
+}
+
+VALUE
+rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size)
+{
+ unsigned int lev = RB_GC_VM_LOCK();
+ rb_gc_vm_barrier();
+
+ // Check if we should trigger GC before allocating
+ maybe_gc(objspace_ptr);
+
+ // Ensure minimum allocation size of BASE_SLOT_SIZE
+ alloc_size = heap_sizes[rb_gc_impl_heap_id_for_size(objspace_ptr, alloc_size)];
+
+ // Allocate memory for the object
+ VALUE *mem = malloc(alloc_size);
+ if (!mem) rb_bug("FIXME: malloc failed");
+
+ // Initialize the object
+ VALUE obj = (VALUE)mem;
+ RBASIC(obj)->flags = flags;
+ *((VALUE *)&RBASIC(obj)->klass) = klass;
+
+ // Register the new object in our tracking table
+ wbcheck_register_object(objspace_ptr, obj, alloc_size, wb_protected);
+
+ // Add this object to the list of objects that need initial reference capture
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ wbcheck_object_list_append(objspace->objects_to_capture, obj);
+
+ RB_GC_VM_UNLOCK(lev);
+ return obj;
+}
+
+size_t
+rb_gc_impl_obj_slot_size(VALUE obj)
+{
+ unsigned int lev = RB_GC_VM_LOCK();
+
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj);
+ size_t result = info->alloc_size;
+
+ RB_GC_VM_UNLOCK(lev);
+ return result;
+}
+
+size_t
+rb_gc_impl_heap_id_for_size(void *objspace_ptr, size_t size)
+{
+ for (int i = 0; i < HEAP_COUNT; i++) {
+ if (size <= heap_sizes[i]) return i;
+ }
+ rb_bug("size too big");
+}
+
+bool
+rb_gc_impl_size_allocatable_p(size_t size)
+{
+ // Only allow sizes up to the largest heap size
+ return size <= MAX_HEAP_SIZE;
+}
+
+// Malloc
+void *
+rb_gc_impl_malloc(void *objspace_ptr, size_t size, bool gc_allowed)
+{
+ if (gc_allowed) {
+ lock_and_maybe_gc(objspace_ptr);
+ }
+ return malloc(size);
+}
+
+void *
+rb_gc_impl_calloc(void *objspace_ptr, size_t size, bool gc_allowed)
+{
+ if (gc_allowed) {
+ lock_and_maybe_gc(objspace_ptr);
+ }
+ return calloc(1, size);
+}
+
+void *
+rb_gc_impl_realloc(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed)
+{
+ if (gc_allowed) {
+ lock_and_maybe_gc(objspace_ptr);
+ }
+ return realloc(ptr, new_size);
+}
+
+void
+rb_gc_impl_free(void *objspace_ptr, void *ptr, size_t old_size)
+{
+ free(ptr);
+}
+
+void
+rb_gc_impl_adjust_memory_usage(void *objspace_ptr, ssize_t diff)
+{
+ // For wbcheck, we don't track memory usage
+}
+
+// Marking
+static void
+gc_mark(rb_wbcheck_objspace_t *objspace, VALUE obj)
+{
+ WBCHECK_DEBUG("wbcheck: gc_mark called\n");
+ wbcheck_debug_obj_info_dump(obj);
+
+ if (RB_SPECIAL_CONST_P(obj)) return;
+
+ switch (objspace->phase) {
+ case WBCHECK_PHASE_SNAPSHOT:
+ // Collecting references during verification
+ GC_ASSERT(objspace->current_refs);
+ wbcheck_object_list_append(objspace->current_refs, obj);
+ break;
+ case WBCHECK_PHASE_FULL_GC:
+ // Marking during full GC
+ wbcheck_mark_gray(objspace, obj);
+ break;
+ case WBCHECK_PHASE_MUTATOR:
+ // Should not be called during mutator phase
+ rb_bug("wbcheck: gc_mark called during mutator phase");
+ break;
+ }
+}
+
+void
+rb_gc_impl_mark(void *objspace_ptr, VALUE obj)
+{
+ rb_wbcheck_objspace_t *objspace = objspace_ptr;
+ gc_mark(objspace, obj);
+}
+
+void
+rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr)
+{
+ rb_wbcheck_objspace_t *objspace = objspace_ptr;
+ gc_mark(objspace, *ptr);
+}
+
+void
+rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj)
+{
+ rb_wbcheck_objspace_t *objspace = objspace_ptr;
+ gc_mark(objspace, obj);
+}
+
+void
+rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj)
+{
+ rb_wbcheck_objspace_t *objspace = objspace_ptr;
+
+ if (!rb_gc_impl_pointer_to_heap_p(objspace_ptr, (void *)obj)) return;
+
+ switch (objspace->phase) {
+ case WBCHECK_PHASE_SNAPSHOT:
+ // We don't know if this is actually a reference or just a value
+ // that looks like one, so we can't expect a write barrier for it.
+ // Keep it separate from the verifiable refs, but retain it so full
+ // GC can mark the target gray if it does turn out to be live.
+ if (!objspace->current_maybe_refs) {
+ objspace->current_maybe_refs = wbcheck_object_list_init();
+ }
+ wbcheck_object_list_append(objspace->current_maybe_refs, obj);
+ break;
+ case WBCHECK_PHASE_FULL_GC:
+ wbcheck_mark_gray(objspace, obj);
+ break;
+ case WBCHECK_PHASE_MUTATOR:
+ rb_bug("wbcheck: rb_gc_impl_mark_maybe called during mutator phase");
+ break;
+ }
+}
+
+// Weak references
+void
+rb_gc_impl_declare_weak_references(void *objspace_ptr, VALUE obj)
+{
+ FL_SET_RAW(obj, RUBY_FL_WEAK_REFERENCE);
+}
+
+bool
+rb_gc_impl_handle_weak_references_alive_p(void *objspace_ptr, VALUE obj)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+
+ st_data_t value;
+ if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) {
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value;
+ return info->color != WBCHECK_COLOR_WHITE;
+ }
+
+ return false;
+}
+
+// Compaction
+void
+rb_gc_impl_register_pinning_obj(void *objspace_ptr, VALUE obj)
+{
+ /* no-op */
+}
+
+bool
+rb_gc_impl_object_moved_p(void *objspace_ptr, VALUE obj)
+{
+ // Stub implementation
+ return false;
+}
+
+VALUE
+rb_gc_impl_location(void *objspace_ptr, VALUE value)
+{
+ // Stub implementation
+ return Qnil;
+}
+
+// Write barriers
+void
+rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b)
+{
+ if (RB_SPECIAL_CONST_P(b)) return;
+
+ unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
+
+ rb_wbcheck_objspace_t *objspace = objspace_ptr;
+
+ // Get the object info for the parent object (a)
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(a);
+
+ // Only record the write barrier if we have a valid snapshot
+ if (info->state != WBCHECK_STATE_CLEAR) {
+ RUBY_ASSERT(info->gc_mark_snapshot);
+
+ // Initialize writebarrier_children list if it doesn't exist
+ if (!info->writebarrier_children) {
+ info->writebarrier_children = wbcheck_object_list_init();
+ }
+
+ // Add the new reference to the write barrier children list
+ wbcheck_object_list_append(info->writebarrier_children, b);
+
+ WBCHECK_DEBUG("wbcheck: write barrier recorded reference from %p to %p\n", (void *)a, (void *)b);
+
+ // If verification after write barrier is enabled, queue the object for verification
+ if (wbcheck_verify_after_wb_enabled && info->state != WBCHECK_STATE_DIRTY) {
+ WBCHECK_DEBUG("wbcheck: queueing object for verification after write barrier\n");
+ info->state = WBCHECK_STATE_DIRTY; // Mark as dirty
+ wbcheck_object_list_append(objspace->objects_to_verify, a);
+ }
+ } else {
+ WBCHECK_DEBUG("wbcheck: write barrier skipped (snapshot not initialized) from %p to %p\n", (void *)a, (void *)b);
+ }
+
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
+}
+
+void
+rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj)
+{
+ WBCHECK_DEBUG("wbcheck: writebarrier_unprotect called on object %p\n", (void *)obj);
+
+ unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
+
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj);
+ info->wb_protected = false;
+
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
+}
+
+void
+rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
+{
+ WBCHECK_DEBUG("wbcheck: writebarrier_remember called on object %p\n", (void *)obj);
+
+ unsigned int lev = RB_GC_VM_LOCK_NO_BARRIER();
+
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj);
+
+ // Clear existing references since they may be stale
+ if (info->state != WBCHECK_STATE_CLEAR) {
+ RUBY_ASSERT(info->gc_mark_snapshot);
+ wbcheck_object_list_free(info->gc_mark_snapshot);
+ info->gc_mark_snapshot = NULL;
+
+ wbcheck_object_list_free(info->mark_maybe_snapshot);
+ info->mark_maybe_snapshot = NULL;
+
+ // Only re-add to objects_to_capture if it had previous snapshot
+ // (new objects don't need to be re-added since they'll be captured at allocation)
+ wbcheck_object_list_append(objspace->objects_to_capture, obj);
+
+ // Also clear write barrier children
+ if (info->writebarrier_children) {
+ wbcheck_object_list_free(info->writebarrier_children);
+ info->writebarrier_children = NULL;
+ }
+
+ // Reset to clear state
+ info->state = WBCHECK_STATE_CLEAR;
+ }
+ RUBY_ASSERT(!info->gc_mark_snapshot);
+ RUBY_ASSERT(!info->mark_maybe_snapshot);
+ RUBY_ASSERT(!info->writebarrier_children);
+
+ RB_GC_VM_UNLOCK_NO_BARRIER(lev);
+}
+
+// Heap walking
+struct wbcheck_foreach_data {
+ int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data);
+ void *data;
+};
+
+static int
+wbcheck_foreach_object_i(st_data_t key, st_data_t val, st_data_t arg)
+{
+ VALUE obj = (VALUE)key;
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)val;
+ struct wbcheck_foreach_data *foreach_data = (struct wbcheck_foreach_data *)arg;
+
+ return foreach_data->callback(obj, info, foreach_data->data);
+}
+
+static void
+wbcheck_foreach_object(rb_wbcheck_objspace_t *objspace, int (*callback)(VALUE obj, rb_wbcheck_object_info_t *info, void *data), void *data)
+{
+ struct wbcheck_foreach_data foreach_data = {
+ .callback = callback,
+ .data = data
+ };
+
+ st_foreach(objspace->object_table, wbcheck_foreach_object_i, (st_data_t)&foreach_data);
+}
+
+// Helper to collect all objects into a snapshot list
+static int
+wbcheck_snapshot_collector(st_data_t key, st_data_t val, st_data_t arg)
+{
+ VALUE obj = (VALUE)key;
+ wbcheck_object_list_t *snapshot = (wbcheck_object_list_t *)arg;
+ wbcheck_object_list_append(snapshot, obj);
+ return ST_CONTINUE;
+}
+
+// Take a snapshot of all objects for safe iteration
+static wbcheck_object_list_t *
+wbcheck_take_object_snapshot(rb_wbcheck_objspace_t *objspace)
+{
+ size_t object_count = st_table_size(objspace->object_table);
+ wbcheck_object_list_t *snapshot = wbcheck_object_list_init_with_capacity(object_count);
+ st_foreach(objspace->object_table, wbcheck_snapshot_collector, (st_data_t)snapshot);
+ return snapshot;
+}
+
+
+void
+rb_gc_impl_each_objects(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ GC_ASSERT(objspace);
+
+ wbcheck_object_list_t *snapshot = wbcheck_take_object_snapshot(objspace);
+
+ for (size_t i = 0; i < snapshot->count; i++) {
+ VALUE obj = snapshot->items[i];
+ st_data_t value;
+ if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) {
+ rb_wbcheck_object_info_t *info = (rb_wbcheck_object_info_t *)value;
+ int result = callback(
+ (void *)obj,
+ (void *)((char *)obj + info->alloc_size),
+ info->alloc_size,
+ data
+ );
+ if (result != 0) break;
+ }
+ }
+
+ wbcheck_object_list_free(snapshot);
+}
+
+void
+rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data), void *data)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ GC_ASSERT(objspace);
+
+ wbcheck_object_list_t *snapshot = wbcheck_take_object_snapshot(objspace);
+
+ for (size_t i = 0; i < snapshot->count; i++) {
+ VALUE obj = snapshot->items[i];
+ st_data_t value;
+ if (st_lookup(objspace->object_table, (st_data_t)obj, &value)) {
+ func(obj, data);
+ }
+ }
+
+ wbcheck_object_list_free(snapshot);
+}
+
+static void
+finalizer_jobs_push(rb_wbcheck_objspace_t *objspace, struct wbcheck_final_job *job)
+{
+ rb_native_mutex_lock(&objspace->finalizer_lock);
+ job->next = objspace->finalizer_jobs;
+ objspace->finalizer_jobs = job;
+ rb_native_mutex_unlock(&objspace->finalizer_lock);
+}
+
+static struct wbcheck_final_job *
+finalizer_jobs_pop(rb_wbcheck_objspace_t *objspace)
+{
+ rb_native_mutex_lock(&objspace->finalizer_lock);
+ struct wbcheck_final_job *job = objspace->finalizer_jobs;
+ if (job) {
+ objspace->finalizer_jobs = job->next;
+ }
+ rb_native_mutex_unlock(&objspace->finalizer_lock);
+ return job;
+}
+
+// Finalizers
+void
+rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), void *data)
+{
+ if (dfree == NULL) return;
+
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+
+ struct wbcheck_final_job *job = malloc(sizeof(struct wbcheck_final_job));
+ job->kind = WBCHECK_FINAL_JOB_DFREE;
+ job->as.dfree.func = dfree;
+ job->as.dfree.data = data;
+
+ finalizer_jobs_push(objspace, job);
+
+ if (!ruby_free_at_exit_p()) {
+ rb_postponed_job_trigger(objspace->finalizer_postponed_job);
+ }
+
+ WBCHECK_DEBUG("wbcheck: made zombie for object %p with dfree function\n", (void *)obj);
+}
+
+VALUE
+rb_gc_impl_define_finalizer(void *objspace_ptr, VALUE obj, VALUE block)
+{
+ unsigned int lev = RB_GC_VM_LOCK();
+
+ (void)objspace_ptr;
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj);
+
+ GC_ASSERT(!OBJ_FROZEN(obj));
+
+ RBASIC(obj)->flags |= FL_FINALIZE;
+
+ VALUE table = info->finalizers;
+ VALUE result = block;
+
+ if (!table) {
+ /* First finalizer for this object - store object ID as first element */
+ table = rb_ary_new3(2, rb_obj_id(obj), block);
+ rb_obj_hide(table);
+ info->finalizers = table;
+ } else {
+ /* Check for duplicate finalizers (skip index 0 which is object ID) */
+ long len = RARRAY_LEN(table);
+ long i;
+
+ for (i = 1; i < len; i++) {
+ VALUE recv = RARRAY_AREF(table, i);
+ if (rb_equal(recv, block)) {
+ result = recv; /* Duplicate found, return existing */
+ goto unlock_and_return;
+ }
+ }
+
+ rb_ary_push(table, block);
+ }
+
+unlock_and_return:
+ RB_GC_VM_UNLOCK(lev);
+ return result;
+}
+
+void
+rb_gc_impl_undefine_finalizer(void *objspace_ptr, VALUE obj)
+{
+ unsigned int lev = RB_GC_VM_LOCK();
+
+ (void)objspace_ptr;
+ rb_wbcheck_object_info_t *info = wbcheck_get_object_info(obj);
+
+ GC_ASSERT(!OBJ_FROZEN(obj));
+
+ info->finalizers = 0;
+ FL_UNSET(obj, FL_FINALIZE);
+
+ RB_GC_VM_UNLOCK(lev);
+}
+
+void
+rb_gc_impl_copy_finalizer(void *objspace_ptr, VALUE dest, VALUE obj)
+{
+ (void)objspace_ptr;
+
+ if (!FL_TEST(obj, FL_FINALIZE)) return;
+
+ unsigned int lev = RB_GC_VM_LOCK();
+
+ rb_wbcheck_object_info_t *src_info = wbcheck_get_object_info(obj);
+ rb_wbcheck_object_info_t *dest_info = wbcheck_get_object_info(dest);
+
+ if (src_info->finalizers) {
+ VALUE table = rb_ary_dup(src_info->finalizers);
+ RARRAY_ASET(table, 0, rb_obj_id(dest));
+ rb_obj_hide(table);
+ dest_info->finalizers = table;
+ FL_SET(dest, FL_FINALIZE);
+ }
+
+ RB_GC_VM_UNLOCK(lev);
+}
+
+static VALUE
+wbcheck_get_final(long i, void *data)
+{
+ VALUE table = (VALUE)data;
+
+ return RARRAY_AREF(table, i + 1);
+}
+
+static void
+make_final_job(rb_wbcheck_objspace_t *objspace, VALUE obj, VALUE finalizer_array)
+{
+ RUBY_ASSERT(RB_FL_TEST(obj, FL_FINALIZE));
+ RUBY_ASSERT(RB_BUILTIN_TYPE(finalizer_array) == T_ARRAY);
+
+ RB_FL_UNSET(obj, FL_FINALIZE);
+
+ struct wbcheck_final_job *job = malloc(sizeof(struct wbcheck_final_job));
+ job->kind = WBCHECK_FINAL_JOB_FINALIZE;
+ job->as.finalize.finalizer_array = finalizer_array;
+
+ finalizer_jobs_push(objspace, job);
+}
+
+static void
+gc_run_finalizers(void *data)
+{
+ rb_wbcheck_objspace_t *objspace = data;
+
+ rb_gc_set_pending_interrupt();
+
+ struct wbcheck_final_job *job;
+ while ((job = finalizer_jobs_pop(objspace)) != NULL) {
+ switch (job->kind) {
+ case WBCHECK_FINAL_JOB_DFREE:
+ job->as.dfree.func(job->as.dfree.data);
+ break;
+ case WBCHECK_FINAL_JOB_FINALIZE: {
+ VALUE finalizer_array = job->as.finalize.finalizer_array;
+
+ rb_gc_run_obj_finalizer(
+ RARRAY_AREF(finalizer_array, 0),
+ RARRAY_LEN(finalizer_array) - 1,
+ wbcheck_get_final,
+ (void *)finalizer_array
+ );
+
+ RB_GC_GUARD(finalizer_array);
+ break;
+ }
+ }
+
+ free(job);
+ }
+
+ rb_gc_unset_pending_interrupt();
+}
+
+static void
+wbcheck_run_finalizers_for_object(VALUE obj, rb_wbcheck_object_info_t *info)
+{
+ if (info->finalizers) {
+ VALUE table = info->finalizers;
+ long count = RARRAY_LEN(table) - 1;
+ rb_gc_run_obj_finalizer(RARRAY_AREF(table, 0), count, wbcheck_get_final, (void *)table);
+ FL_UNSET(obj, FL_FINALIZE);
+ }
+ info->finalizers = 0;
+}
+
+static int
+wbcheck_shutdown_call_finalizer_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data)
+{
+ wbcheck_run_finalizers_for_object(obj, info);
+ return ST_CONTINUE; /* Keep iterating through all objects */
+}
+
+static int
+wbcheck_verify_all_references_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data)
+{
+ void *objspace_ptr = data;
+ wbcheck_verify_object_references(objspace_ptr, obj);
+ return ST_CONTINUE;
+}
+
+static int
+wbcheck_update_all_snapshots_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data)
+{
+ void *objspace_ptr = data;
+
+ // For wb_protected objects, do full verification if they have a snapshot
+ if (info->wb_protected && info->state != WBCHECK_STATE_CLEAR) {
+ wbcheck_verify_object_references(objspace_ptr, obj);
+ } else {
+ // For CLEAR objects (wb_protected or not) and non-wb_protected objects, just take a new snapshot
+ wbcheck_object_list_t *current_refs = wbcheck_collect_references_from_object(obj, info);
+ wbcheck_object_list_free(info->gc_mark_snapshot);
+ info->gc_mark_snapshot = current_refs;
+ info->state = WBCHECK_STATE_MARKED;
+ }
+
+ return ST_CONTINUE;
+}
+
+static int
+wbcheck_shutdown_finalizer_callback(VALUE obj, rb_wbcheck_object_info_t *info, void *data)
+{
+ void *objspace_ptr = data;
+
+ if (rb_gc_shutdown_call_finalizer_p(obj)) {
+ WBCHECK_DEBUG("wbcheck: finalizing object during shutdown: %p\n", (void *)obj);
+ rb_gc_obj_free_vm_weak_references(obj);
+ if (rb_gc_obj_free(objspace_ptr, obj)) {
+ RBASIC(obj)->flags = 0;
+ }
+ }
+
+ return ST_CONTINUE;
+}
+
+
+void
+rb_gc_impl_shutdown_call_finalizer(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = objspace_ptr;
+
+ // Call all finalizers for all objects using our shared iteration helper
+ wbcheck_foreach_object(objspace, wbcheck_shutdown_call_finalizer_callback, NULL);
+
+ // After all finalizers have been called, verify all object references
+ unsigned int verify_lev = RB_GC_VM_LOCK();
+ WBCHECK_DEBUG("wbcheck: verifying references for all objects after finalizers\n");
+ wbcheck_foreach_object(objspace, wbcheck_verify_all_references_callback, objspace_ptr);
+ WBCHECK_DEBUG("wbcheck: finished verifying all object references\n");
+ RB_GC_VM_UNLOCK(verify_lev);
+
+ // Print summary and exit with error code if violations were found
+ if (objspace->missed_write_barrier_parents > 0 || objspace->missed_write_barrier_children > 0) {
+ fprintf(stderr, "WBCHECK SUMMARY: Found %zu objects with missed write barriers (%zu total violations)\n",
+ objspace->missed_write_barrier_parents, objspace->missed_write_barrier_children);
+
+
+ exit(1); // Exit with error code to indicate violations were found
+ } else {
+ WBCHECK_DEBUG("wbcheck: no write barrier violations detected\n");
+ }
+
+ // Call rb_gc_obj_free on objects that need shutdown finalization (File, Data with dfree, etc.)
+ unsigned int lev = RB_GC_VM_LOCK();
+ WBCHECK_DEBUG("wbcheck: calling rb_gc_obj_free on objects that need shutdown finalization\n");
+ wbcheck_foreach_object(objspace, wbcheck_shutdown_finalizer_callback, objspace_ptr);
+ WBCHECK_DEBUG("wbcheck: finished calling rb_gc_obj_free\n");
+
+ // Run any pending finalizer jobs (dfree functions)
+ WBCHECK_DEBUG("wbcheck: running pending finalizer jobs\n");
+ gc_run_finalizers(objspace);
+ WBCHECK_DEBUG("wbcheck: finished running finalizer jobs\n");
+ RB_GC_VM_UNLOCK(lev);
+}
+
+// Forking
+void
+rb_gc_impl_before_fork(void *objspace_ptr)
+{
+ // Stub implementation
+}
+
+void
+rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
+{
+ // Stub implementation
+}
+
+// Statistics
+void
+rb_gc_impl_set_measure_total_time(void *objspace_ptr, VALUE flag)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ objspace->measure_total_time = RTEST(flag);
+}
+
+bool
+rb_gc_impl_get_measure_total_time(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ return objspace->measure_total_time;
+}
+
+unsigned long long
+rb_gc_impl_get_total_time(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ return objspace->measure_total_time ? objspace->simulated_gc_count : 0;
+}
+
+size_t
+rb_gc_impl_gc_count(void *objspace_ptr)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ if (objspace) {
+ return objspace->simulated_gc_count;
+ }
+ return 0;
+}
+
+VALUE
+rb_gc_impl_latest_gc_info(void *objspace_ptr, VALUE key)
+{
+ // Stub implementation
+ return Qnil;
+}
+
+VALUE
+rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
+{
+ rb_wbcheck_objspace_t *objspace = (rb_wbcheck_objspace_t *)objspace_ptr;
+ GC_ASSERT(objspace);
+
+ VALUE hash = Qnil, key = Qnil;
+
+ if (RB_TYPE_P(hash_or_sym, T_HASH)) {
+ hash = hash_or_sym;
+ }
+ else if (SYMBOL_P(hash_or_sym)) {
+ key = hash_or_sym;
+ }
+ else {
+ rb_bug("non-hash or symbol given");
+ }
+
+#define SET(name, attr) \
+ if (key == ID2SYM(rb_intern(#name))) \
+ return SIZET2NUM(attr); \
+ else if (hash != Qnil) \
+ rb_hash_aset(hash, ID2SYM(rb_intern(#name)), SIZET2NUM(attr));
+
+ /* Pretend each GC takes 1ms; :time is reported in milliseconds. */
+ SET(count, objspace->simulated_gc_count);
+ SET(time, objspace->measure_total_time ? objspace->simulated_gc_count : 0);
+ SET(tracked_objects, st_table_size(objspace->object_table));
+#undef SET
+
+ if (!NIL_P(key)) {
+ rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
+ }
+
+ rb_hash_aset(hash, ID2SYM(rb_intern("gc_implementation")), rb_str_new_cstr("wbcheck"));
+
+ return hash;
+}
+
+VALUE
+rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym)
+{
+ if (FIXNUM_P(heap_name) && SYMBOL_P(hash_or_sym)) {
+ int heap_idx = FIX2INT(heap_name);
+ if (heap_idx < 0 || heap_idx >= HEAP_COUNT) {
+ rb_raise(rb_eArgError, "size pool index out of range");
+ }
+
+ if (hash_or_sym == ID2SYM(rb_intern("slot_size"))) {
+ return SIZET2NUM(heap_sizes[heap_idx]);
+ }
+
+ return Qundef;
+ }
+
+ if (RB_TYPE_P(hash_or_sym, T_HASH)) {
+ return hash_or_sym;
+ }
+
+ return Qundef;
+}
+
+const char *
+rb_gc_impl_active_gc_name(void)
+{
+ // Stub implementation
+ return "wbcheck";
+}
+
+// Miscellaneous
+#define WBCHECK_OBJECT_METADATA_ENTRY_COUNT 2
+static struct rb_gc_object_metadata_entry object_metadata_entries[WBCHECK_OBJECT_METADATA_ENTRY_COUNT + 1];
+
+struct rb_gc_object_metadata_entry *
+rb_gc_impl_object_metadata(void *objspace_ptr, VALUE obj)
+{
+ static ID ID_object_id, ID_shareable;
+
+ if (!ID_object_id) {
+ ID_object_id = rb_intern("object_id");
+ ID_shareable = rb_intern("shareable");
+ }
+
+ size_t n = 0;
+
+#define SET_ENTRY(na, v) do { \
+ GC_ASSERT(n < WBCHECK_OBJECT_METADATA_ENTRY_COUNT); \
+ object_metadata_entries[n].name = ID_##na; \
+ object_metadata_entries[n].val = v; \
+ n++; \
+} while (0)
+
+ if (rb_obj_id_p(obj)) SET_ENTRY(object_id, rb_obj_id(obj));
+ if (FL_TEST(obj, FL_SHAREABLE)) SET_ENTRY(shareable, Qtrue);
+#undef SET_ENTRY
+
+ object_metadata_entries[n].name = 0;
+ object_metadata_entries[n].val = 0;
+
+ return object_metadata_entries;
+}
+
+bool
+rb_gc_impl_pointer_to_heap_p(void *objspace_ptr, const void *ptr)
+{
+ GC_ASSERT(wbcheck_global_objspace);
+
+ unsigned int lev = RB_GC_VM_LOCK();
+
+ // Check if this pointer exists in our object tracking table
+ st_data_t value;
+ bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)ptr, &value);
+
+ RB_GC_VM_UNLOCK(lev);
+ return result;
+}
+
+bool
+rb_gc_impl_garbage_object_p(void *objspace_ptr, VALUE obj)
+{
+ unsigned int lev = RB_GC_VM_LOCK();
+
+ // Check if this pointer exists in our object tracking table
+ st_data_t value;
+ bool result = st_lookup(wbcheck_global_objspace->object_table, (st_data_t)obj, &value);
+
+ RB_GC_VM_UNLOCK(lev);
+ return !result;
+}
+
+void
+rb_gc_impl_set_event_hook(void *objspace_ptr, const rb_event_flag_t event)
+{
+ // Stub implementation
+}
+
+void
+rb_gc_impl_copy_attributes(void *objspace_ptr, VALUE dest, VALUE obj)
+{
+ rb_wbcheck_object_info_t *src_info = wbcheck_get_object_info(obj);
+
+ if (!src_info->wb_protected) {
+ rb_gc_impl_writebarrier_unprotect(objspace_ptr, dest);
+ }
+ rb_gc_impl_copy_finalizer(objspace_ptr, dest, obj);
+}
+
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-03 22:37:29 +0000