diff options
Diffstat (limited to 'gc.c')
| -rw-r--r-- | gc.c | 1591 |
1 files changed, 977 insertions, 614 deletions
@@ -11,9 +11,6 @@ **********************************************************************/ -#define rb_data_object_alloc rb_data_object_alloc -#define rb_data_typed_object_alloc rb_data_typed_object_alloc - #include "ruby/internal/config.h" #ifdef _WIN32 # include "ruby/ruby.h" @@ -82,7 +79,6 @@ #undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */ #include "constant.h" -#include "darray.h" #include "debug_counter.h" #include "eval_intern.h" #include "gc/gc.h" @@ -103,6 +99,7 @@ #include "internal/object.h" #include "internal/proc.h" #include "internal/rational.h" +#include "internal/re.h" #include "internal/sanitizers.h" #include "internal/struct.h" #include "internal/symbol.h" @@ -127,10 +124,19 @@ #include "vm_callinfo.h" #include "ractor_core.h" #include "yjit.h" +#include "zjit.h" #include "builtin.h" #include "shape.h" +// TODO: Don't export this function in modular GC, instead MMTk should figure out +// how to combine GC thread backtrace with mutator thread backtrace. +void +rb_gc_print_backtrace(void) +{ + rb_print_backtrace(stderr); +} + unsigned int rb_gc_vm_lock(const char *file, int line) { @@ -179,7 +185,6 @@ rb_gc_vm_barrier(void) rb_vm_barrier(); } -#if USE_MODULAR_GC void * rb_gc_get_ractor_newobj_cache(void) { @@ -193,35 +198,6 @@ rb_gc_initialize_vm_context(struct rb_gc_vm_context *context) context->ec = GET_EC(); } -void -rb_gc_worker_thread_set_vm_context(struct rb_gc_vm_context *context) -{ - rb_native_mutex_lock(&context->lock); - - GC_ASSERT(rb_current_execution_context(false) == NULL); - -#ifdef RB_THREAD_LOCAL_SPECIFIER - rb_current_ec_set(context->ec); -#else - native_tls_set(ruby_current_ec_key, context->ec); -#endif -} - -void -rb_gc_worker_thread_unset_vm_context(struct rb_gc_vm_context *context) -{ - rb_native_mutex_unlock(&context->lock); - - GC_ASSERT(rb_current_execution_context(true) == context->ec); - -#ifdef RB_THREAD_LOCAL_SPECIFIER - rb_current_ec_set(NULL); -#else - native_tls_set(ruby_current_ec_key, NULL); -#endif -} -#endif - bool rb_gc_event_hook_required_p(rb_event_flag_t event) { @@ -233,10 +209,33 @@ rb_gc_event_hook(VALUE obj, rb_event_flag_t event) { if (LIKELY(!rb_gc_event_hook_required_p(event))) return; - rb_execution_context_t *ec = GET_EC(); + rb_execution_context_t *ec = rb_gc_get_ec(); if (!ec->cfp) return; +#if USE_MODULAR_GC + bool gc_thread_p = false; + if (!GET_EC()) { + gc_thread_p = true; + +# ifdef RB_THREAD_LOCAL_SPECIFIER + rb_current_ec_set(ec); +# else + native_tls_set(ruby_current_ec_key, ec); +# endif + } +#endif + EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, obj); + +#if USE_MODULAR_GC + if (gc_thread_p) { +# ifdef RB_THREAD_LOCAL_SPECIFIER + rb_current_ec_set(NULL); +# else + native_tls_set(ruby_current_ec_key, NULL); +# endif + } +#endif } void * @@ -245,7 +244,6 @@ rb_gc_get_objspace(void) return GET_VM()->gc.objspace; } - void rb_gc_ractor_newobj_cache_foreach(void (*func)(void *cache, void *data), void *data) { @@ -289,6 +287,7 @@ rb_gc_run_obj_finalizer(VALUE objid, long count, VALUE (*callback)(long i, void saved.finished = 0; saved.final = Qundef; + ASSERT_vm_unlocking(); rb_ractor_ignore_belonging(true); EC_PUSH_TAG(ec); enum ruby_tag_type state = EC_EXEC_TAG(); @@ -337,7 +336,10 @@ rb_gc_shutdown_call_finalizer_p(VALUE obj) { switch (BUILTIN_TYPE(obj)) { case T_DATA: - if (!ruby_free_at_exit_p() && (!DATA_PTR(obj) || !RDATA(obj)->dfree)) return false; + if (!ruby_free_at_exit_p()) { + if (!RDATA(obj)->type) return false; + if (!rbimpl_typeddata_embedded_p(obj) && !RTYPEDDATA(obj)->data) return false; + } if (rb_obj_is_thread(obj)) return false; if (rb_obj_is_mutex(obj)) return false; if (rb_obj_is_fiber(obj)) return false; @@ -361,31 +363,18 @@ rb_gc_shutdown_call_finalizer_p(VALUE obj) } } -uint32_t -rb_gc_get_shape(VALUE obj) -{ - return (uint32_t)rb_obj_shape_id(obj); -} - void -rb_gc_set_shape(VALUE obj, uint32_t shape_id) -{ - rb_obj_set_shape_id(obj, (uint32_t)shape_id); -} - -uint32_t -rb_gc_rebuild_shape(VALUE obj, size_t heap_id) +rb_gc_obj_changed_pool(VALUE obj, size_t heap_id) { RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT)); - return (uint32_t)rb_shape_transition_heap(obj, heap_id); + RBASIC_SET_SHAPE_ID(obj, rb_obj_shape_transition_heap(obj, heap_id)); } void rb_vm_update_references(void *ptr); #define rb_setjmp(env) RUBY_SETJMP(env) #define rb_jmp_buf rb_jmpbuf_t -#undef rb_data_object_wrap #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) #define MAP_ANONYMOUS MAP_ANON @@ -400,7 +389,7 @@ void rb_vm_update_references(void *ptr); #define RMOVED(obj) ((struct RMoved *)(obj)) #define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \ - if (rb_gc_impl_object_moved_p((_objspace), (VALUE)(_thing))) { \ + if (gc_object_moved_p_internal((_objspace), (VALUE)(_thing))) { \ *(_type *)&(_thing) = (_type)gc_location_internal(_objspace, (VALUE)_thing); \ } \ } while (0) @@ -482,21 +471,21 @@ rb_malloc_grow_capa(size_t current, size_t type_size) return new_capacity; } -static inline struct rbimpl_size_mul_overflow_tag +static inline struct rbimpl_size_overflow_tag size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */ { - struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y); - struct rbimpl_size_mul_overflow_tag u = rbimpl_size_add_overflow(t.right, z); - return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left, u.right }; + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y); + struct rbimpl_size_overflow_tag u = rbimpl_size_add_overflow(t.result, z); + return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed, u.result }; } -static inline struct rbimpl_size_mul_overflow_tag +static inline struct rbimpl_size_overflow_tag size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */ { - struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y); - struct rbimpl_size_mul_overflow_tag u = rbimpl_size_mul_overflow(z, w); - struct rbimpl_size_mul_overflow_tag v = rbimpl_size_add_overflow(t.right, u.right); - return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left || v.left, v.right }; + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y); + struct rbimpl_size_overflow_tag u = rbimpl_size_mul_overflow(z, w); + struct rbimpl_size_overflow_tag v = rbimpl_size_add_overflow(t.result, u.result); + return (struct rbimpl_size_overflow_tag) { t.overflowed || u.overflowed || v.overflowed, v.result }; } PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3); @@ -504,9 +493,9 @@ PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3); static inline size_t size_mul_or_raise(size_t x, size_t y, VALUE exc) { - struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y); - if (LIKELY(!t.left)) { - return t.right; + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(x, y); + if (LIKELY(!t.overflowed)) { + return t.result; } else if (rb_during_gc()) { rb_memerror(); /* or...? */ @@ -530,9 +519,9 @@ rb_size_mul_or_raise(size_t x, size_t y, VALUE exc) static inline size_t size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc) { - struct rbimpl_size_mul_overflow_tag t = size_mul_add_overflow(x, y, z); - if (LIKELY(!t.left)) { - return t.right; + struct rbimpl_size_overflow_tag t = size_mul_add_overflow(x, y, z); + if (LIKELY(!t.overflowed)) { + return t.result; } else if (rb_during_gc()) { rb_memerror(); /* or...? */ @@ -557,9 +546,9 @@ rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc) static inline size_t size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc) { - struct rbimpl_size_mul_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w); - if (LIKELY(!t.left)) { - return t.right; + struct rbimpl_size_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w); + if (LIKELY(!t.overflowed)) { + return t.result; } else if (rb_during_gc()) { rb_memerror(); /* or...? */ @@ -589,6 +578,7 @@ rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val) #endif static const char *obj_type_name(VALUE obj); +static st_table *id2ref_tbl; #include "gc/default/default.c" #if USE_MODULAR_GC && !defined(HAVE_DLOPEN) @@ -619,15 +609,16 @@ typedef struct gc_function_map { void (*config_set)(void *objspace_ptr, VALUE hash); void (*stress_set)(void *objspace_ptr, VALUE flag); VALUE (*stress_get)(void *objspace_ptr); + struct rb_gc_vm_context *(*get_vm_context)(void *objspace_ptr); // Object allocation - VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size); + VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size); size_t (*obj_slot_size)(VALUE obj); size_t (*heap_id_for_size)(void *objspace_ptr, size_t size); bool (*size_allocatable_p)(size_t size); // Malloc - void *(*malloc)(void *objspace_ptr, size_t size); - void *(*calloc)(void *objspace_ptr, size_t size); - void *(*realloc)(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size); + void *(*malloc)(void *objspace_ptr, size_t size, bool gc_allowed); + void *(*calloc)(void *objspace_ptr, size_t size, bool gc_allowed); + void *(*realloc)(void *objspace_ptr, void *ptr, size_t new_size, size_t old_size, bool gc_allowed); void (*free)(void *objspace_ptr, void *ptr, size_t old_size); void (*adjust_memory_usage)(void *objspace_ptr, ssize_t diff); // Marking @@ -635,9 +626,11 @@ typedef struct gc_function_map { void (*mark_and_move)(void *objspace_ptr, VALUE *ptr); void (*mark_and_pin)(void *objspace_ptr, VALUE obj); void (*mark_maybe)(void *objspace_ptr, VALUE obj); - void (*mark_weak)(void *objspace_ptr, VALUE *ptr); - void (*remove_weak)(void *objspace_ptr, VALUE parent_obj, VALUE *ptr); + // Weak references + void (*declare_weak_references)(void *objspace_ptr, VALUE obj); + bool (*handle_weak_references_alive_p)(void *objspace_ptr, VALUE obj); // Compaction + void (*register_pinning_obj)(void *objspace_ptr, VALUE obj); bool (*object_moved_p)(void *objspace_ptr, VALUE obj); VALUE (*location)(void *objspace_ptr, VALUE value); // Write barriers @@ -704,7 +697,7 @@ ruby_modular_gc_init(void) break; default: fprintf(stderr, "Only alphanumeric, dash, and underscore is allowed in "RUBY_GC_LIBRARY"\n"); - exit(1); + exit(EXIT_FAILURE); } } @@ -751,19 +744,21 @@ ruby_modular_gc_init(void) handle = dlopen(gc_so_path, RTLD_LAZY | RTLD_GLOBAL); if (!handle) { fprintf(stderr, "ruby_modular_gc_init: Shared library %s cannot be opened: %s\n", gc_so_path, dlerror()); - exit(1); + exit(EXIT_FAILURE); } gc_functions.modular_gc_loaded_p = true; } + unsigned int err_count = 0; + # define load_modular_gc_func(name) do { \ if (handle) { \ const char *func_name = "rb_gc_impl_" #name; \ gc_functions.name = dlsym(handle, func_name); \ if (!gc_functions.name) { \ fprintf(stderr, "ruby_modular_gc_init: %s function not exported by library %s\n", func_name, gc_so_path); \ - exit(1); \ + err_count++; \ } \ } \ else { \ @@ -793,6 +788,7 @@ ruby_modular_gc_init(void) load_modular_gc_func(config_get); load_modular_gc_func(stress_set); load_modular_gc_func(stress_get); + load_modular_gc_func(get_vm_context); // Object allocation load_modular_gc_func(new_obj); load_modular_gc_func(obj_slot_size); @@ -809,9 +805,11 @@ ruby_modular_gc_init(void) load_modular_gc_func(mark_and_move); load_modular_gc_func(mark_and_pin); load_modular_gc_func(mark_maybe); - load_modular_gc_func(mark_weak); - load_modular_gc_func(remove_weak); + // Weak references + load_modular_gc_func(declare_weak_references); + load_modular_gc_func(handle_weak_references_alive_p); // Compaction + load_modular_gc_func(register_pinning_obj); load_modular_gc_func(object_moved_p); load_modular_gc_func(location); // Write barriers @@ -846,6 +844,11 @@ ruby_modular_gc_init(void) load_modular_gc_func(set_event_hook); load_modular_gc_func(copy_attributes); + if (err_count > 0) { + fprintf(stderr, "ruby_modular_gc_init: found %u missing exports in library %s\n", err_count, gc_so_path); + exit(EXIT_FAILURE); + } + # undef load_modular_gc_func rb_gc_functions = gc_functions; @@ -873,6 +876,7 @@ ruby_modular_gc_init(void) # define rb_gc_impl_config_set rb_gc_functions.config_set # define rb_gc_impl_stress_set rb_gc_functions.stress_set # define rb_gc_impl_stress_get rb_gc_functions.stress_get +# define rb_gc_impl_get_vm_context rb_gc_functions.get_vm_context // Object allocation # define rb_gc_impl_new_obj rb_gc_functions.new_obj # define rb_gc_impl_obj_slot_size rb_gc_functions.obj_slot_size @@ -889,9 +893,11 @@ ruby_modular_gc_init(void) # define rb_gc_impl_mark_and_move rb_gc_functions.mark_and_move # define rb_gc_impl_mark_and_pin rb_gc_functions.mark_and_pin # define rb_gc_impl_mark_maybe rb_gc_functions.mark_maybe -# define rb_gc_impl_mark_weak rb_gc_functions.mark_weak -# define rb_gc_impl_remove_weak rb_gc_functions.remove_weak +// Weak references +# define rb_gc_impl_declare_weak_references rb_gc_functions.declare_weak_references +# define rb_gc_impl_handle_weak_references_alive_p rb_gc_functions.handle_weak_references_alive_p // Compaction +# define rb_gc_impl_register_pinning_obj rb_gc_functions.register_pinning_obj # define rb_gc_impl_object_moved_p rb_gc_functions.object_moved_p # define rb_gc_impl_location rb_gc_functions.location // Write barriers @@ -971,60 +977,159 @@ rb_gc_obj_slot_size(VALUE obj) } static inline void -gc_validate_pc(void) +gc_validate_pc(VALUE obj) { #if RUBY_DEBUG + // IMEMOs and objects without a class (e.g managed id table) are not traceable + if (RB_TYPE_P(obj, T_IMEMO) || !CLASS_OF(obj)) return; + rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *cfp = ec->cfp; - if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && cfp->pc) { - RUBY_ASSERT(cfp->pc >= ISEQ_BODY(cfp->iseq)->iseq_encoded); - RUBY_ASSERT(cfp->pc <= ISEQ_BODY(cfp->iseq)->iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size); + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && CFP_PC(cfp)) { + const VALUE *iseq_encoded = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; + const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(CFP_ISEQ(cfp))->iseq_size; + RUBY_ASSERT(CFP_PC(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); + RUBY_ASSERT(CFP_PC(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); } #endif } -static inline VALUE -newobj_of(rb_ractor_t *cr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t size) +NOINLINE(static void gc_newobj_hook(VALUE obj)); +static void +gc_newobj_hook(VALUE obj) { - VALUE obj = rb_gc_impl_new_obj(rb_gc_get_objspace(), cr->newobj_cache, klass, flags, v1, v2, v3, wb_protected, size); + int lev = RB_GC_VM_LOCK_NO_BARRIER(); + { + size_t slot_size = rb_gc_obj_slot_size(obj); + memset((char *)obj + sizeof(struct RBasic), 0, slot_size - sizeof(struct RBasic)); + + /* We must disable GC here because the callback could call xmalloc + * which could potentially trigger a GC, and a lot of code is unsafe + * to trigger a GC right after an object has been allocated because + * they perform initialization for the object and assume that the + * GC does not trigger before then. */ + bool gc_disabled = RTEST(rb_gc_disable_no_rest()); + { + rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_NEWOBJ); + } + if (!gc_disabled) rb_gc_enable(); + } + RB_GC_VM_UNLOCK_NO_BARRIER(lev); +} + +VALUE +rb_newobj(rb_execution_context_t *ec, VALUE klass, VALUE flags, shape_id_t shape_id, bool wb_protected, size_t size) +{ + GC_ASSERT((flags & FL_WB_PROTECTED) == 0); + rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + VALUE obj = rb_gc_impl_new_obj(rb_gc_get_objspace(), cr->newobj_cache, klass, flags, wb_protected, size); + +#if RACTOR_CHECK_MODE + void rb_ractor_setup_belonging(VALUE obj); + rb_ractor_setup_belonging(obj); +#endif + + RBASIC_SET_SHAPE_ID_NO_CHECKS(obj, shape_id); - gc_validate_pc(); + gc_validate_pc(obj); if (UNLIKELY(rb_gc_event_hook_required_p(RUBY_INTERNAL_EVENT_NEWOBJ))) { - unsigned int lev; - RB_VM_LOCK_ENTER_CR_LEV(cr, &lev); - { - memset((char *)obj + RVALUE_SIZE, 0, rb_gc_obj_slot_size(obj) - RVALUE_SIZE); - - /* We must disable GC here because the callback could call xmalloc - * which could potentially trigger a GC, and a lot of code is unsafe - * to trigger a GC right after an object has been allocated because - * they perform initialization for the object and assume that the - * GC does not trigger before then. */ - bool gc_disabled = RTEST(rb_gc_disable_no_rest()); - { - rb_gc_event_hook(obj, RUBY_INTERNAL_EVENT_NEWOBJ); - } - if (!gc_disabled) rb_gc_enable(); - } - RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev); + gc_newobj_hook(obj); } +#if RGENGC_CHECK_MODE +# ifndef GC_DEBUG_SLOT_FILL_SPECIAL_VALUE +# define GC_DEBUG_SLOT_FILL_SPECIAL_VALUE 255 +# endif + + memset( + (void *)(obj + sizeof(struct RBasic)), + GC_DEBUG_SLOT_FILL_SPECIAL_VALUE, + rb_gc_obj_slot_size(obj) - sizeof(struct RBasic) + ); +#endif + return obj; } VALUE -rb_wb_unprotected_newobj_of(VALUE klass, VALUE flags, size_t size) +rb_ec_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size) { - GC_ASSERT((flags & FL_WB_PROTECTED) == 0); - return newobj_of(GET_RACTOR(), klass, flags, 0, 0, 0, FALSE, size); + VALUE type = flags & T_MASK; + RUBY_ASSERT(type != T_OBJECT); + RUBY_ASSERT(type != T_DATA); + RUBY_ASSERT(type != T_CLASS); + RUBY_ASSERT(type != T_MODULE); + RUBY_ASSERT(type != T_ICLASS); + (void)type; + + return rb_newobj(ec, klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size); } VALUE -rb_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size) +rb_newobj_of_with_shape(VALUE klass, VALUE flags, shape_id_t shape_id, size_t size) { - GC_ASSERT((flags & FL_WB_PROTECTED) == 0); - return newobj_of(rb_ec_ractor_ptr(ec), klass, flags, 0, 0, 0, TRUE, size); + return rb_newobj(GET_EC(), klass, flags, shape_id, true, size); +} + +VALUE +rb_newobj_of(VALUE klass, VALUE flags, size_t size) +{ + return rb_newobj(GET_EC(), klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size); +} + +static +VALUE class_allocate_complex_instance(VALUE klass, uint32_t capacity) +{ + shape_id_t initial_shape_id = rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(sizeof(struct RObject)))); + VALUE obj = rb_newobj_of_with_shape(klass, T_OBJECT, initial_shape_id, sizeof(struct RObject)); + rb_obj_init_complex(obj, rb_st_init_numtable_with_size(capacity)); + return obj; +} + +VALUE +rb_class_allocate_instance(VALUE klass) +{ + uint32_t index_tbl_num_entries = RCLASS_MAX_IV_COUNT(klass); + VALUE obj; + + // Directly start as COMPLEX if we know we're over the limit. + RUBY_ASSERT(rb_shape_tree.max_capacity > 0); + if (RB_UNLIKELY(index_tbl_num_entries > rb_shape_tree.max_capacity)) { + obj = class_allocate_complex_instance(klass, index_tbl_num_entries); + } + else { + size_t size = rb_obj_embedded_size(index_tbl_num_entries); + if (!rb_gc_size_allocatable_p(size)) { + size = sizeof(struct RObject); + } + + // There might be a NEWOBJ tracepoint callback, and it may set fields. + // So the shape must be passed to `NEWOBJ_OF`. + obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(size))), size); + + #if RUBY_DEBUG + VALUE *ptr = ROBJECT_FIELDS(obj); + size_t fields_count = RSHAPE_LEN(RBASIC_SHAPE_ID(obj)); + for (size_t i = fields_count; i < ROBJECT_FIELDS_CAPACITY(obj); i++) { + ptr[i] = Qundef; + } + #endif + } + +#if RUBY_DEBUG + if (rb_obj_class(obj) != rb_class_real(klass)) { + rb_bug("Expected rb_class_allocate_instance to set the class correctly"); + } +#endif + + return obj; +} + +void +rb_gc_register_pinning_obj(VALUE obj) +{ + rb_gc_impl_register_pinning_obj(rb_gc_get_objspace(), obj); } #define UNEXPECTED_NODE(func) \ @@ -1034,27 +1139,16 @@ rb_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, static inline void rb_data_object_check(VALUE klass) { + RUBY_ASSERT(!RCLASS_SINGLETON_P(klass)); if (klass != rb_cObject && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) { rb_undef_alloc_func(klass); rb_warn("undefining the allocator of T_DATA class %"PRIsVALUE, klass); } } -VALUE -rb_data_object_wrap(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree) -{ - RUBY_ASSERT_ALWAYS(dfree != (RUBY_DATA_FUNC)1); - if (klass) rb_data_object_check(klass); - return newobj_of(GET_RACTOR(), klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, !dmark, sizeof(struct RTypedData)); -} - -VALUE -rb_data_object_zalloc(VALUE klass, size_t size, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree) -{ - VALUE obj = rb_data_object_wrap(klass, 0, dmark, dfree); - DATA_PTR(obj) = xcalloc(1, size); - return obj; -} +#define RTYPEDDATA_EMBEDDED_P rbimpl_typeddata_embedded_p +#define RB_DATA_TYPE_EMBEDDABLE_P(type) ((type)->flags & RUBY_TYPED_EMBEDDABLE) +#define RTYPEDDATA_EMBEDDABLE_P(obj) RB_DATA_TYPE_EMBEDDABLE_P(RTYPEDDATA_TYPE(obj)) static VALUE typed_data_alloc(VALUE klass, VALUE typed_flag, void *datap, const rb_data_type_t *type, size_t size) @@ -1062,13 +1156,22 @@ typed_data_alloc(VALUE klass, VALUE typed_flag, void *datap, const rb_data_type_ RBIMPL_NONNULL_ARG(type); if (klass) rb_data_object_check(klass); bool wb_protected = (type->flags & RUBY_FL_WB_PROTECTED) || !type->function.dmark; - return newobj_of(GET_RACTOR(), klass, T_DATA, 0, ((VALUE)type) | IS_TYPED_DATA | typed_flag, (VALUE)datap, wb_protected, size); + VALUE obj = rb_newobj(GET_EC(), klass, T_DATA, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_RDATA, wb_protected, size); + + rb_gc_register_pinning_obj(obj); + + struct RTypedData *data = (struct RTypedData *)obj; + data->fields_obj = 0; + *(VALUE *)&data->type = ((VALUE)type) | typed_flag; + data->data = datap; + + return obj; } VALUE rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type) { - if (UNLIKELY(type->flags & RUBY_TYPED_EMBEDDABLE)) { + if (UNLIKELY(RB_DATA_TYPE_EMBEDDABLE_P(type))) { rb_raise(rb_eTypeError, "Cannot wrap an embeddable TypedData"); } @@ -1078,7 +1181,7 @@ rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type) VALUE rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type) { - if (type->flags & RUBY_TYPED_EMBEDDABLE) { + if (RB_DATA_TYPE_EMBEDDABLE_P(type)) { if (!(type->flags & RUBY_TYPED_FREE_IMMEDIATELY)) { rb_raise(rb_eTypeError, "Embeddable TypedData must be freed immediately"); } @@ -1097,20 +1200,33 @@ rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type } static size_t +ruby_xmalloc_usable_size(void *ptr) +{ +#ifdef HAVE_MALLOC_USABLE_SIZE +#if CALC_EXACT_MALLOC_SIZE + struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1; + return malloc_usable_size(info) - sizeof(struct malloc_obj_info); +#else + return malloc_usable_size(ptr); +#endif +#else + return 0; +#endif +} + +static size_t rb_objspace_data_type_memsize(VALUE obj) { size_t size = 0; - if (RTYPEDDATA_P(obj)) { - const rb_data_type_t *type = RTYPEDDATA_TYPE(obj); - const void *ptr = RTYPEDDATA_GET_DATA(obj); + const void *ptr = RTYPEDDATA_GET_DATA(obj); - if (RTYPEDDATA_TYPE(obj)->flags & RUBY_TYPED_EMBEDDABLE && !RTYPEDDATA_EMBEDDED_P(obj)) { -#ifdef HAVE_MALLOC_USABLE_SIZE - size += malloc_usable_size((void *)ptr); -#endif + if (ptr) { + if (RTYPEDDATA_EMBEDDABLE_P(obj) && !RTYPEDDATA_EMBEDDED_P(obj)) { + size += ruby_xmalloc_usable_size((void *)ptr); } - if (ptr && type->function.dsize) { + const rb_data_type_t *type = RTYPEDDATA_TYPE(obj); + if (type->function.dsize) { size += type->function.dsize(ptr); } } @@ -1121,19 +1237,207 @@ rb_objspace_data_type_memsize(VALUE obj) const char * rb_objspace_data_type_name(VALUE obj) { - if (RTYPEDDATA_P(obj)) { - return RTYPEDDATA_TYPE(obj)->wrap_struct_name; + return RTYPEDDATA_TYPE(obj)->wrap_struct_name; +} + +void +rb_gc_declare_weak_references(VALUE obj) +{ + rb_gc_impl_declare_weak_references(rb_gc_get_objspace(), obj); +} + +bool +rb_gc_handle_weak_references_alive_p(VALUE obj) +{ + if (SPECIAL_CONST_P(obj)) return true; + + return rb_gc_impl_handle_weak_references_alive_p(rb_gc_get_objspace(), obj); +} + +void +rb_gc_handle_weak_references(VALUE obj) +{ + switch (BUILTIN_TYPE(obj)) { + case T_DATA: + { + const rb_data_type_t *type = RTYPEDDATA_TYPE(obj); + + if (type->function.handle_weak_references) { + (type->function.handle_weak_references)(RTYPEDDATA_GET_DATA(obj)); + } + else { + rb_bug( + "rb_gc_handle_weak_references: TypedData %s does not implement handle_weak_references", + RTYPEDDATA_TYPE(obj)->wrap_struct_name + ); + } + } + break; + + case T_IMEMO: { + switch (imemo_type(obj)) { + case imemo_callcache: { + struct rb_callcache *cc = (struct rb_callcache *)obj; + if (cc->klass != Qundef && + (!rb_gc_handle_weak_references_alive_p(cc->klass) || + !rb_gc_handle_weak_references_alive_p((VALUE)cc->cme_))) { + vm_cc_invalidate(cc); + } + break; + } + case imemo_subclasses: { + struct rb_subclasses *subs = (struct rb_subclasses *)obj; + VALUE *entries = rb_imemo_subclasses_entries(obj); + for (uint32_t i = 0; i < subs->count; i++) { + if (entries[i] && !rb_gc_handle_weak_references_alive_p(entries[i])) { + entries[i] = 0; + } + } + break; + } + default: + rb_bug("rb_gc_handle_weak_references: unexpected imemo type"); + } + + break; + } + default: + rb_bug("rb_gc_handle_weak_references: type not supported\n"); } - else { - return 0; +} + +static inline bool +rb_gc_imemo_needs_cleanup_p(VALUE obj) +{ + switch (imemo_type(obj)) { + case imemo_constcache: + case imemo_cref: + case imemo_ifunc: + case imemo_memo: + case imemo_svar: + case imemo_callcache: + case imemo_throw_data: + case imemo_cvar_entry: + return false; + + case imemo_env: + case imemo_ment: + case imemo_iseq: + case imemo_callinfo: + case imemo_cdhash: + return true; + + case imemo_subclasses: + return FL_TEST_RAW(obj, IMEMO_SUBCLASSES_HEAP); + + case imemo_tmpbuf: + return ((rb_imemo_tmpbuf_t *)obj)->ptr != NULL; + + case imemo_fields: + return FL_TEST_RAW(obj, OBJ_FIELD_HEAP) || (id2ref_tbl && rb_obj_shape_has_id(obj)); } + UNREACHABLE_RETURN(true); } -static enum rb_id_table_iterator_result -cvar_table_free_i(VALUE value, void *ctx) +/* + * Returns true if the object requires a full rb_gc_obj_free() call during sweep, + * false if it can be freed quickly without calling destructors or cleanup. + * + * Objects that return false are: + * - Simple embedded objects without external allocations + * - Objects without finalizers + * - Objects without object IDs registered in id2ref + * - Objects without generic instance variables + * + * This is used by the GC sweep fast path to avoid function call overhead + * for the majority of simple objects. + */ +bool +rb_gc_obj_needs_cleanup_p(VALUE obj) { - xfree((void *)value); - return ID_TABLE_CONTINUE; + VALUE flags = RBASIC(obj)->flags; + + if (flags & FL_FINALIZE) return true; + + switch (flags & RUBY_T_MASK) { + case T_IMEMO: + return rb_gc_imemo_needs_cleanup_p(obj); + + case T_DATA: + case T_OBJECT: + case T_STRING: + case T_ARRAY: + case T_HASH: + case T_BIGNUM: + case T_STRUCT: + case T_FLOAT: + case T_RATIONAL: + case T_COMPLEX: + case T_MATCH: + break; + + case T_FILE: + case T_SYMBOL: + case T_CLASS: + case T_ICLASS: + case T_MODULE: + case T_REGEXP: + return true; + } + + shape_id_t shape_id = RBASIC_SHAPE_ID(obj); + if (id2ref_tbl && rb_shape_has_object_id(shape_id)) return true; + + switch (flags & RUBY_T_MASK) { + case T_OBJECT: + if (flags & ROBJECT_HEAP) return true; + return false; + + case T_DATA: + { + uintptr_t type = (uintptr_t)RTYPEDDATA(obj)->type; + if (type & TYPED_DATA_EMBEDDED) { + RUBY_DATA_FUNC dfree = ((const rb_data_type_t *)(type & TYPED_DATA_PTR_MASK))->function.dfree; + if (dfree == RUBY_NEVER_FREE || dfree == RUBY_TYPED_DEFAULT_FREE) { + return false; + } + } + } + return true; + + case T_STRING: + if (flags & (RSTRING_NOEMBED | RSTRING_FSTR)) return true; + return rb_shape_has_fields(shape_id); + + case T_ARRAY: + if (!(flags & RARRAY_EMBED_FLAG)) return true; + return rb_shape_has_fields(shape_id); + + case T_HASH: + if (flags & RHASH_ST_TABLE_FLAG) return true; + return rb_shape_has_fields(shape_id); + + case T_MATCH: + if ((flags & (RMATCH_ONIG | RMATCH_OFFSETS_EXTERNAL)) || USE_DEBUG_COUNTER) return true; + return rb_shape_has_fields(shape_id); + + case T_BIGNUM: + if (!(flags & BIGNUM_EMBED_FLAG)) return true; + return rb_shape_has_fields(shape_id); + + case T_STRUCT: + if (!(flags & RSTRUCT_EMBED_LEN_MASK)) return true; + if (flags & RSTRUCT_GEN_FIELDS) return rb_shape_has_fields(shape_id); + return false; + + case T_FLOAT: + case T_RATIONAL: + case T_COMPLEX: + return rb_shape_has_fields(shape_id); + + default: + UNREACHABLE_RETURN(true); + } } static void @@ -1152,29 +1456,24 @@ make_io_zombie(void *objspace, VALUE obj) static bool rb_data_free(void *objspace, VALUE obj) { - void *data = RTYPEDDATA_P(obj) ? RTYPEDDATA_GET_DATA(obj) : DATA_PTR(obj); + void *data = RTYPEDDATA_GET_DATA(obj); if (data) { int free_immediately = false; void (*dfree)(void *); - if (RTYPEDDATA_P(obj)) { - free_immediately = (RTYPEDDATA_TYPE(obj)->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0; - dfree = RTYPEDDATA_TYPE(obj)->function.dfree; - } - else { - dfree = RDATA(obj)->dfree; - } + free_immediately = (RTYPEDDATA_TYPE(obj)->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0; + dfree = RTYPEDDATA_TYPE(obj)->function.dfree; if (dfree) { if (dfree == RUBY_DEFAULT_FREE) { - if (!RTYPEDDATA_P(obj) || !RTYPEDDATA_EMBEDDED_P(obj)) { + if (!RTYPEDDATA_EMBEDDED_P(obj)) { xfree(data); RB_DEBUG_COUNTER_INC(obj_data_xfree); } } else if (free_immediately) { (*dfree)(data); - if (RTYPEDDATA_TYPE(obj)->flags & RUBY_TYPED_EMBEDDABLE && !RTYPEDDATA_EMBEDDED_P(obj)) { + if (RTYPEDDATA_EMBEDDABLE_P(obj) && !RTYPEDDATA_EMBEDDED_P(obj)) { xfree(data); } @@ -1200,48 +1499,19 @@ struct classext_foreach_args { }; static void -classext_free(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg) +classext_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) { - struct rb_id_table *tbl; struct classext_foreach_args *args = (struct classext_foreach_args *)arg; - rb_id_table_free(RCLASSEXT_M_TBL(ext)); - - if (!RCLASSEXT_SHARED_CONST_TBL(ext) && (tbl = RCLASSEXT_CONST_TBL(ext)) != NULL) { - rb_free_const_table(tbl); - } - if ((tbl = RCLASSEXT_CVC_TBL(ext)) != NULL) { - rb_id_table_foreach_values(tbl, cvar_table_free_i, NULL); - rb_id_table_free(tbl); - } - rb_class_classext_free_subclasses(ext, args->klass); - if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) { - RUBY_ASSERT(is_prime); // superclasses should only be used on prime - xfree(RCLASSEXT_SUPERCLASSES(ext)); - } - if (!is_prime) { // the prime classext will be freed with RClass - xfree(ext); - } + rb_class_classext_free(args->klass, ext, is_prime); } static void -classext_iclass_free(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg) +classext_iclass_free(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) { struct classext_foreach_args *args = (struct classext_foreach_args *)arg; - if (RCLASSEXT_ICLASS_IS_ORIGIN(ext) && !RCLASSEXT_ICLASS_ORIGIN_SHARED_MTBL(ext)) { - /* Method table is not shared for origin iclasses of classes */ - rb_id_table_free(RCLASSEXT_M_TBL(ext)); - } - if (RCLASSEXT_CALLABLE_M_TBL(ext) != NULL) { - rb_id_table_free(RCLASSEXT_CALLABLE_M_TBL(ext)); - } - - rb_class_classext_free_subclasses(ext, args->klass); - - if (!is_prime) { // the prime classext will be freed with RClass - xfree(ext); - } + rb_iclass_classext_free(args->klass, ext, is_prime); } bool @@ -1264,20 +1534,25 @@ rb_gc_obj_free(void *objspace, VALUE obj) switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - if (rb_shape_obj_too_complex_p(obj)) { - RB_DEBUG_COUNTER_INC(obj_obj_too_complex); - st_free_table(ROBJECT_FIELDS_HASH(obj)); - } - else if (RBASIC(obj)->flags & ROBJECT_EMBED) { - RB_DEBUG_COUNTER_INC(obj_obj_embed); + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(obj)) { + RB_DEBUG_COUNTER_INC(obj_obj_complex); + st_free_table(ROBJECT_FIELDS_HASH(obj)); + } + else { + SIZED_FREE_N(ROBJECT(obj)->as.heap.fields, ROBJECT_FIELDS_CAPACITY(obj)); + RB_DEBUG_COUNTER_INC(obj_obj_ptr); + } } else { - xfree(ROBJECT(obj)->as.heap.fields); - RB_DEBUG_COUNTER_INC(obj_obj_ptr); + RB_DEBUG_COUNTER_INC(obj_obj_embed); } break; case T_MODULE: case T_CLASS: +#if USE_ZJIT + rb_zjit_klass_free(obj); +#endif args.klass = obj; rb_class_classext_foreach(obj, classext_free, (void *)&args); if (RCLASS_CLASSEXT_TBL(obj)) { @@ -1347,29 +1622,32 @@ rb_gc_obj_free(void *objspace, VALUE obj) break; case T_MATCH: { - rb_matchext_t *rm = RMATCH_EXT(obj); + struct RMatch *rm = RMATCH(obj); #if USE_DEBUG_COUNTER - if (rm->regs.num_regs >= 8) { + if (rm->num_regs >= 8) { RB_DEBUG_COUNTER_INC(obj_match_ge8); } - else if (rm->regs.num_regs >= 4) { + else if (rm->num_regs >= 4) { RB_DEBUG_COUNTER_INC(obj_match_ge4); } - else if (rm->regs.num_regs >= 1) { + else if (rm->num_regs >= 1) { RB_DEBUG_COUNTER_INC(obj_match_under4); } #endif - onig_region_free(&rm->regs, 0); - xfree(rm->char_offset); + if (FL_TEST_RAW(obj, RMATCH_ONIG)) { + onig_region_free(&rm->as.onig, 0); + } + SIZED_FREE_N(rm->char_offset, rm->char_offset_num_allocated); RB_DEBUG_COUNTER_INC(obj_match_ptr); } break; case T_FILE: if (RFILE(obj)->fptr) { - make_io_zombie(objspace, obj); + bool closed = rb_io_fptr_finalize_closed(RFILE(obj)->fptr); + if (!closed) make_io_zombie(objspace, obj); RB_DEBUG_COUNTER_INC(obj_file_ptr); - return FALSE; + return closed; } break; case T_RATIONAL: @@ -1397,7 +1675,7 @@ rb_gc_obj_free(void *objspace, VALUE obj) case T_BIGNUM: if (!BIGNUM_EMBED_P(obj) && BIGNUM_DIGITS(obj)) { - xfree(BIGNUM_DIGITS(obj)); + SIZED_FREE_N(BIGNUM_DIGITS(obj), BIGNUM_LEN(obj)); RB_DEBUG_COUNTER_INC(obj_bignum_ptr); } else { @@ -1415,7 +1693,7 @@ rb_gc_obj_free(void *objspace, VALUE obj) RB_DEBUG_COUNTER_INC(obj_struct_embed); } else { - xfree((void *)RSTRUCT(obj)->as.heap.ptr); + SIZED_FREE_N(RSTRUCT(obj)->as.heap.ptr, RSTRUCT(obj)->as.heap.len); RB_DEBUG_COUNTER_INC(obj_struct_ptr); } break; @@ -1535,34 +1813,26 @@ os_obj_of(VALUE of) * Ruby process. If <i>module</i> is specified, calls the block * for only those classes or modules that match (or are a subclass of) * <i>module</i>. Returns the number of objects found. Immediate - * objects (<code>Fixnum</code>s, <code>Symbol</code>s - * <code>true</code>, <code>false</code>, and <code>nil</code>) are - * never returned. In the example below, #each_object returns both - * the numbers we defined and several constants defined in the Math - * module. + * objects (such as <code>Fixnum</code>s, static <code>Symbol</code>s + * <code>true</code>, <code>false</code> and <code>nil</code>) are + * never returned. * * If no block is given, an enumerator is returned instead. * - * a = 102.7 - * b = 95 # Won't be returned - * c = 12345678987654321 - * count = ObjectSpace.each_object(Numeric) {|x| p x } + * Job = Class.new + * jobs = [Job.new, Job.new] + * count = ObjectSpace.each_object(Job) {|x| p x } * puts "Total count: #{count}" * * <em>produces:</em> * - * 12345678987654321 - * 102.7 - * 2.71828182845905 - * 3.14159265358979 - * 2.22044604925031e-16 - * 1.7976931348623157e+308 - * 2.2250738585072e-308 - * Total count: 7 + * #<Job:0x000000011d6cbbf0> + * #<Job:0x000000011d6cbc68> + * Total count: 2 * - * Due to a current known Ractor implementation issue, this method will not yield - * Ractor-unshareable objects in multi-Ractor mode (when - * <code>Ractor.new</code> has been called within the process at least once). + * Due to a current Ractor implementation issue, this method does not yield + * Ractor-unshareable objects when the process is in multi-Ractor mode. Multi-ractor + * mode is enabled when <code>Ractor.new</code> has been called for the first time. * See https://bugs.ruby-lang.org/issues/19387 for more information. * * a = 12345678987654321 # shareable @@ -1586,10 +1856,12 @@ os_each_obj(int argc, VALUE *argv, VALUE os) /* * call-seq: - * ObjectSpace.undefine_finalizer(obj) + * ObjectSpace.undefine_finalizer(obj) -> obj * - * Removes all finalizers for <i>obj</i>. + * Removes all finalizers registered for +obj+ with + * ObjectSpace.define_finalizer, and returns +obj+. * + * Does nothing if +obj+ has no finalizers. */ static VALUE @@ -1635,63 +1907,68 @@ rb_gc_copy_finalizer(VALUE dest, VALUE obj) /* * call-seq: - * ObjectSpace.define_finalizer(obj, aProc=proc()) + * ObjectSpace.define_finalizer(obj) {|id| ... } -> array + * ObjectSpace.define_finalizer(obj, finalizer) -> array * - * Adds <i>aProc</i> as a finalizer, to be called after <i>obj</i> - * was destroyed. The object ID of the <i>obj</i> will be passed - * as an argument to <i>aProc</i>. If <i>aProc</i> is a lambda or - * method, make sure it can be called with a single argument. + * Adds a new finalizer for +obj+ that is called when +obj+ is destroyed + * by the garbage collector or when Ruby shuts down (which ever comes first). * - * The return value is an array <code>[0, aProc]</code>. + * With a block given, uses the block as the callback. Without a block given, + * uses a callable object +finalizer+ as the callback. The callback is called + * when +obj+ is destroyed with a single argument +id+ which is the object + * ID of +obj+ (see Object#object_id). * - * The two recommended patterns are to either create the finaliser proc - * in a non-instance method where it can safely capture the needed state, - * or to use a custom callable object that stores the needed state - * explicitly as instance variables. + * The return value is an array <code>[0, callback]</code>, where +callback+ + * is a Proc created from the block if one was given or +finalizer+ otherwise. * - * class Foo - * def initialize(data_needed_for_finalization) - * ObjectSpace.define_finalizer(self, self.class.create_finalizer(data_needed_for_finalization)) - * end + * Note that defining a finalizer in an instance method of the object may prevent + * the object from being garbage collected since if the block or +finalizer+ refers + * to +obj+ then +obj+ will never be reclaimed by the garbage collector. For example, + * the following script demonstrates the issue: * - * def self.create_finalizer(data_needed_for_finalization) - * proc { - * puts "finalizing #{data_needed_for_finalization}" - * } + * class Foo + * def define_final + * ObjectSpace.define_finalizer(self) do |id| + * puts "Running finalizer for #{id}!" + * end * end * end * - * class Bar - * class Remover - * def initialize(data_needed_for_finalization) - * @data_needed_for_finalization = data_needed_for_finalization - * end + * obj = Foo.new + * obj.define_final * - * def call(id) - * puts "finalizing #{@data_needed_for_finalization}" - * end + * There are two patterns to solve this issue: + * + * - Create the finalizer in a non-instance method so it can safely capture + * the needed state: + * + * class Foo + * def define_final + * ObjectSpace.define_finalizer(self, self.class.create_finalizer) * end * - * def initialize(data_needed_for_finalization) - * ObjectSpace.define_finalizer(self, Remover.new(data_needed_for_finalization)) + * def self.create_finalizer + * proc do |id| + * puts "Running finalizer for #{id}!" + * end * end * end * - * Note that if your finalizer references the object to be - * finalized it will never be run on GC, although it will still be - * run at exit. You will get a warning if you capture the object - * to be finalized as the receiver of the finalizer. + * - Use a callable object: * - * class CapturesSelf - * def initialize(name) - * ObjectSpace.define_finalizer(self, proc { - * # this finalizer will only be run on exit - * puts "finalizing #{name}" - * }) + * class Foo + * class Finalizer + * def call(id) + * puts "Running finalizer for #{id}!" + * end + * end + * + * def define_final + * ObjectSpace.define_finalizer(self, Finalizer.new) * end * end * - * Also note that finalization can be unpredictable and is never guaranteed + * Note that finalization can be unpredictable and is never guaranteed * to be run except on exit. */ @@ -1752,7 +2029,6 @@ rb_gc_pointer_to_heap_p(VALUE obj) #define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1) #define LAST_OBJECT_ID() (object_id_counter * OBJ_ID_INCREMENT) static VALUE id2ref_value = 0; -static st_table *id2ref_tbl = NULL; #if SIZEOF_SIZE_T == SIZEOF_LONG_LONG static size_t object_id_counter = 1; @@ -1814,7 +2090,7 @@ id2ref_tbl_mark(void *data) // It's very unlikely, but if enough object ids were generated, keys may be T_BIGNUM rb_mark_set(table); } - // We purposedly don't mark values, as they are weak references. + // We purposely don't mark values, as they are weak references. // rb_gc_obj_free_vm_weak_references takes care of cleaning them up. } @@ -1867,8 +2143,8 @@ static inline VALUE object_id_get(VALUE obj, shape_id_t shape_id) { VALUE id; - if (rb_shape_too_complex_p(shape_id)) { - id = rb_obj_field_get(obj, ROOT_TOO_COMPLEX_WITH_OBJ_ID); + if (rb_shape_complex_p(shape_id)) { + id = rb_obj_field_get(obj, ROOT_COMPLEX_WITH_OBJ_ID); } else { id = rb_obj_field_get(obj, rb_shape_object_id(shape_id)); @@ -1894,16 +2170,18 @@ object_id0(VALUE obj) return object_id_get(obj, shape_id); } - shape_id_t object_id_shape_id = rb_shape_transition_object_id(obj); + shape_id_t object_id_shape_id = rb_obj_shape_transition_object_id(obj); id = generate_next_object_id(); rb_obj_field_set(obj, object_id_shape_id, 0, id); RUBY_ASSERT(RBASIC_SHAPE_ID(obj) == object_id_shape_id); - RUBY_ASSERT(rb_shape_obj_has_id(obj)); + RUBY_ASSERT(rb_obj_shape_has_id(obj)); if (RB_UNLIKELY(id2ref_tbl)) { - st_insert(id2ref_tbl, (st_data_t)id, (st_data_t)obj); + RB_VM_LOCKING() { + st_insert(id2ref_tbl, (st_data_t)id, (st_data_t)obj); + } } return id; } @@ -1914,8 +2192,8 @@ object_id(VALUE obj) switch (BUILTIN_TYPE(obj)) { case T_CLASS: case T_MODULE: - // With namespaces, classes and modules have different fields - // in different namespaces, so we cannot store the object id + // With Ruby Box, classes and modules have different fields + // in different boxes, so we cannot store the object id // in fields. return class_object_id(obj); case T_IMEMO: @@ -1950,13 +2228,13 @@ build_id2ref_i(VALUE obj, void *data) break; case T_IMEMO: RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (IMEMO_TYPE_P(obj, imemo_fields) && rb_shape_obj_has_id(obj)) { + if (IMEMO_TYPE_P(obj, imemo_fields) && rb_obj_shape_has_id(obj)) { st_insert(id2ref_tbl, rb_obj_id(obj), rb_imemo_fields_owner(obj)); } break; case T_OBJECT: RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (rb_shape_obj_has_id(obj)) { + if (rb_obj_shape_has_id(obj)) { st_insert(id2ref_tbl, rb_obj_id(obj), obj); } break; @@ -2057,9 +2335,10 @@ obj_free_object_id(VALUE obj) void rb_gc_obj_free_vm_weak_references(VALUE obj) { + ASSUME(!RB_SPECIAL_CONST_P(obj)); obj_free_object_id(obj); - if (rb_obj_exivar_p(obj)) { + if (rb_obj_gen_fields_p(obj)) { rb_free_generic_ivar(obj); } @@ -2074,15 +2353,6 @@ rb_gc_obj_free_vm_weak_references(VALUE obj) break; case T_IMEMO: switch (imemo_type(obj)) { - case imemo_callcache: { - const struct rb_callcache *cc = (const struct rb_callcache *)obj; - - if (vm_cc_refinement_p(cc)) { - rb_vm_delete_cc_refinement(cc); - } - - break; - } case imemo_callinfo: rb_vm_ci_free((const struct rb_callinfo *)obj); break; @@ -2118,14 +2388,9 @@ rb_gc_obj_free_vm_weak_references(VALUE obj) static VALUE id2ref(VALUE objid) { -#if SIZEOF_LONG == SIZEOF_VOIDP -#define NUM2PTR(x) NUM2ULONG(x) -#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP -#define NUM2PTR(x) NUM2ULL(x) -#endif objid = rb_to_int(objid); if (FIXNUM_P(objid) || rb_big_size(objid) <= SIZEOF_VOIDP) { - VALUE ptr = NUM2PTR(objid); + VALUE ptr = (VALUE)NUM2PTR(objid); if (SPECIAL_CONST_P(ptr)) { if (ptr == Qtrue) return Qtrue; if (ptr == Qfalse) return Qfalse; @@ -2236,7 +2501,7 @@ rb_obj_id(VALUE obj) bool rb_obj_id_p(VALUE obj) { - return !RB_TYPE_P(obj, T_IMEMO) && rb_shape_obj_has_id(obj); + return !RB_TYPE_P(obj, T_IMEMO) && rb_obj_shape_has_id(obj); } /* @@ -2251,6 +2516,9 @@ rb_gc_before_updating_jit_code(void) #if USE_YJIT rb_yjit_mark_all_writeable(); #endif +#if USE_ZJIT + rb_zjit_mark_all_writable(); +#endif } /* @@ -2264,10 +2532,13 @@ rb_gc_after_updating_jit_code(void) #if USE_YJIT rb_yjit_mark_all_executable(); #endif +#if USE_ZJIT + rb_zjit_mark_all_executable(); +#endif } static void -classext_memsize(rb_classext_t *ext, bool prime, VALUE namespace, void *arg) +classext_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) { size_t *size = (size_t *)arg; size_t s = 0; @@ -2275,9 +2546,6 @@ classext_memsize(rb_classext_t *ext, bool prime, VALUE namespace, void *arg) if (RCLASSEXT_M_TBL(ext)) { s += rb_id_table_memsize(RCLASSEXT_M_TBL(ext)); } - if (RCLASSEXT_CVC_TBL(ext)) { - s += rb_id_table_memsize(RCLASSEXT_CVC_TBL(ext)); - } if (RCLASSEXT_CONST_TBL(ext)) { s += rb_id_table_memsize(RCLASSEXT_CONST_TBL(ext)); } @@ -2291,7 +2559,7 @@ classext_memsize(rb_classext_t *ext, bool prime, VALUE namespace, void *arg) } static void -classext_superclasses_memsize(rb_classext_t *ext, bool prime, VALUE namespace, void *arg) +classext_superclasses_memsize(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) { size_t *size = (size_t *)arg; size_t array_size; @@ -2313,11 +2581,13 @@ rb_obj_memsize_of(VALUE obj) switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - if (rb_shape_obj_too_complex_p(obj)) { - size += rb_st_memsize(ROBJECT_FIELDS_HASH(obj)); - } - else if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) { - size += ROBJECT_FIELDS_CAPACITY(obj) * sizeof(VALUE); + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(obj)) { + size += rb_st_memsize(ROBJECT_FIELDS_HASH(obj)); + } + else { + size += ROBJECT_FIELDS_CAPACITY(obj) * sizeof(VALUE); + } } break; case T_MODULE: @@ -2355,8 +2625,10 @@ rb_obj_memsize_of(VALUE obj) break; case T_MATCH: { - rb_matchext_t *rm = RMATCH_EXT(obj); - size += onig_region_memsize(&rm->regs); + struct RMatch *rm = RMATCH(obj); + if (FL_TEST_RAW(obj, RMATCH_ONIG)) { + size += onig_region_memsize(&rm->as.onig); + } size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated; } break; @@ -2387,9 +2659,8 @@ rb_obj_memsize_of(VALUE obj) break; case T_STRUCT: - if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 && - RSTRUCT(obj)->as.heap.ptr) { - size += sizeof(VALUE) * RSTRUCT_LEN(obj); + if (RSTRUCT_EMBED_LEN(obj) == 0) { + size += sizeof(VALUE) * RSTRUCT_LEN_RAW(obj); } break; @@ -2437,34 +2708,39 @@ count_objects_i(VALUE obj, void *d) /* * call-seq: - * ObjectSpace.count_objects([result_hash]) -> hash + * ObjectSpace.count_objects(result_hash = {}) -> hash * - * Counts all objects grouped by type. + * Counts the number of objects, grouped by type. * - * It returns a hash, such as: - * { - * :TOTAL=>10000, - * :FREE=>3011, - * :T_OBJECT=>6, - * :T_CLASS=>404, - * # ... - * } + * It returns a hash that looks like: * - * The contents of the returned hash are implementation specific. - * It may be changed in future. + * { + * TOTAL: 10000, + * FREE: 3011, + * T_OBJECT: 6, + * T_CLASS: 404, + * # ... + * } * - * The keys starting with +:T_+ means live objects. + * The contents of the returned hash are implementation specific and + * may be changed in future versions without notice. + * + * The keys starting with +:T_+ are live objects of a particular type. * For example, +:T_ARRAY+ is the number of arrays. - * +:FREE+ means object slots which is not used now. - * +:TOTAL+ means sum of above. + * + * The key +:FREE+ is the number of object slots which are empty. + * + * The key +:TOTAL+ is the total number of slots (which is the sum of + * all of the other values). * * If the optional argument +result_hash+ is given, - * it is overwritten and returned. This is intended to avoid probe effect. + * it is overwritten and returned. + * This is intended to avoid the probe effect. * * h = {} * ObjectSpace.count_objects(h) * puts h - * # => { :TOTAL=>10000, :T_CLASS=>158280, :T_MODULE=>20672, :T_STRING=>527249 } + * # => { TOTAL: 10000, T_CLASS: 158280, T_MODULE: 20672, T_STRING: 527249 } * * This method is only expected to work on C Ruby. * @@ -2490,6 +2766,10 @@ count_objects(int argc, VALUE *argv, VALUE os) types[i] = type_sym(i); } + // Same as type_sym, we need to create all key symbols in advance + VALUE total = ID2SYM(rb_intern("TOTAL")); + VALUE free = ID2SYM(rb_intern("FREE")); + rb_gc_impl_each_object(rb_gc_get_objspace(), count_objects_i, &data); if (NIL_P(hash)) { @@ -2498,8 +2778,8 @@ count_objects(int argc, VALUE *argv, VALUE os) else if (!RHASH_EMPTY_P(hash)) { rb_hash_stlike_foreach(hash, set_zero, hash); } - rb_hash_aset(hash, ID2SYM(rb_intern("TOTAL")), SIZET2NUM(data.total)); - rb_hash_aset(hash, ID2SYM(rb_intern("FREE")), SIZET2NUM(data.freed)); + rb_hash_aset(hash, total, SIZET2NUM(data.total)); + rb_hash_aset(hash, free, SIZET2NUM(data.freed)); for (size_t i = 0; i <= T_MASK; i++) { if (data.counts[i]) { @@ -2648,29 +2928,6 @@ rb_gc_mark_maybe(VALUE obj) gc_mark_maybe_internal(obj); } -void -rb_gc_mark_weak(VALUE *ptr) -{ - if (RB_SPECIAL_CONST_P(*ptr)) return; - - rb_vm_t *vm = GET_VM(); - void *objspace = vm->gc.objspace; - if (LIKELY(vm->gc.mark_func_data == NULL)) { - GC_ASSERT(rb_gc_impl_during_gc_p(objspace)); - - rb_gc_impl_mark_weak(objspace, ptr); - } - else { - GC_ASSERT(!rb_gc_impl_during_gc_p(objspace)); - } -} - -void -rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr) -{ - rb_gc_impl_remove_weak(rb_gc_get_objspace(), parent_obj, ptr); -} - ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void each_location(register const VALUE *x, register long n, void (*cb)(VALUE, void *), void *data)); static void each_location(register const VALUE *x, register long n, void (*cb)(VALUE, void *), void *data) @@ -2803,8 +3060,10 @@ mark_const_entry_i(VALUE value, void *objspace) { const rb_const_entry_t *ce = (const rb_const_entry_t *)value; - gc_mark_internal(ce->value); - gc_mark_internal(ce->file); + if (!rb_gc_checking_shareable()) { + gc_mark_internal(ce->value); + gc_mark_internal(ce->file); // TODO: ce->file should be shareable? + } return ID_TABLE_CONTINUE; } @@ -2815,26 +3074,6 @@ mark_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl) rb_id_table_foreach_values(tbl, mark_const_entry_i, objspace); } -static enum rb_id_table_iterator_result -mark_cvc_tbl_i(VALUE cvc_entry, void *objspace) -{ - struct rb_cvar_class_tbl_entry *entry; - - entry = (struct rb_cvar_class_tbl_entry *)cvc_entry; - - RUBY_ASSERT(entry->cref == 0 || (BUILTIN_TYPE((VALUE)entry->cref) == T_IMEMO && IMEMO_TYPE_P(entry->cref, imemo_cref))); - gc_mark_internal((VALUE)entry->cref); - - return ID_TABLE_CONTINUE; -} - -static void -mark_cvc_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl) -{ - if (!tbl) return; - rb_id_table_foreach_values(tbl, mark_cvc_tbl_i, objspace); -} - #if STACK_GROW_DIRECTION < 0 #define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START) #elif STACK_GROW_DIRECTION > 0 @@ -2865,6 +3104,16 @@ gc_mark_machine_stack_location_maybe(VALUE obj, void *data) #endif } +static bool +gc_object_moved_p_internal(void *objspace, VALUE obj) +{ + if (SPECIAL_CONST_P(obj)) { + return false; + } + + return rb_gc_impl_object_moved_p(objspace, obj); +} + static VALUE gc_location_internal(void *objspace, VALUE value) { @@ -2997,16 +3246,37 @@ rb_mark_tbl_no_pin(st_table *tbl) gc_mark_tbl_no_pin(tbl); } +void +rb_gc_mark_set_no_pin(st_table *tbl) +{ + if (!tbl || tbl->num_entries == 0) return; + + st_foreach(tbl, gc_mark_set_no_pin_i, 0); +} + static bool gc_declarative_marking_p(const rb_data_type_t *type) { return (type->flags & RUBY_TYPED_DECL_MARKING) != 0; } +rb_execution_context_t * +rb_gc_get_ec(void) +{ + void *objspace = rb_gc_get_objspace(); + + if (RB_LIKELY(rb_gc_impl_during_gc_p(objspace))) { + return rb_gc_impl_get_vm_context(objspace)->ec; + } + else { + return GET_EC(); + } +} + void rb_gc_mark_roots(void *objspace, const char **categoryp) { - rb_execution_context_t *ec = GET_EC(); + rb_execution_context_t *ec = rb_gc_get_ec(); rb_vm_t *vm = rb_ec_vm_ptr(ec); #define MARK_CHECKPOINT(category) do { \ @@ -3031,6 +3301,14 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) } #endif +#if USE_ZJIT + void rb_zjit_root_mark(void); + if (rb_zjit_enabled_p) { + MARK_CHECKPOINT("ZJIT"); + rb_zjit_root_mark(); + } +#endif + MARK_CHECKPOINT("machine_context"); mark_current_machine_context(ec); @@ -3048,7 +3326,7 @@ struct gc_mark_classext_foreach_arg { }; static void -gc_mark_classext_module(rb_classext_t *ext, bool prime, VALUE namespace, void *arg) +gc_mark_classext_module(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) { struct gc_mark_classext_foreach_arg *foreach_arg = (struct gc_mark_classext_foreach_arg *)arg; rb_objspace_t *objspace = foreach_arg->objspace; @@ -3057,18 +3335,26 @@ gc_mark_classext_module(rb_classext_t *ext, bool prime, VALUE namespace, void *a gc_mark_internal(RCLASSEXT_SUPER(ext)); } mark_m_tbl(objspace, RCLASSEXT_M_TBL(ext)); - gc_mark_internal(RCLASSEXT_FIELDS_OBJ(ext)); + + if (!rb_gc_checking_shareable()) { + // unshareable + gc_mark_internal(RCLASSEXT_FIELDS_OBJ(ext)); + gc_mark_internal(RCLASSEXT_CVC_TBL(ext)); + } + if (!RCLASSEXT_SHARED_CONST_TBL(ext) && RCLASSEXT_CONST_TBL(ext)) { mark_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext)); } mark_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext)); gc_mark_internal(RCLASSEXT_CC_TBL(ext)); - mark_cvc_tbl(objspace, RCLASSEXT_CVC_TBL(ext)); + if (RCLASSEXT_SUBCLASSES(ext)) { + gc_mark_internal(RCLASSEXT_SUBCLASSES(ext)); + } gc_mark_internal(RCLASSEXT_CLASSPATH(ext)); } static void -gc_mark_classext_iclass(rb_classext_t *ext, bool prime, VALUE namespace, void *arg) +gc_mark_classext_iclass(rb_classext_t *ext, bool prime, VALUE box_value, void *arg) { struct gc_mark_classext_foreach_arg *foreach_arg = (struct gc_mark_classext_foreach_arg *)arg; rb_objspace_t *objspace = foreach_arg->objspace; @@ -3084,16 +3370,27 @@ gc_mark_classext_iclass(rb_classext_t *ext, bool prime, VALUE namespace, void *a } mark_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext)); gc_mark_internal(RCLASSEXT_CC_TBL(ext)); + if (RCLASSEXT_SUBCLASSES(ext)) { + gc_mark_internal(RCLASSEXT_SUBCLASSES(ext)); + } } #define TYPED_DATA_REFS_OFFSET_LIST(d) (size_t *)(uintptr_t)RTYPEDDATA_TYPE(d)->function.dmark void +rb_gc_move_obj_during_marking(VALUE from, VALUE to) +{ + if (rb_obj_using_gen_fields_table_p(to)) { + rb_mark_generic_ivar(from); + } +} + +void rb_gc_mark_children(void *objspace, VALUE obj) { struct gc_mark_classext_foreach_arg foreach_args; - if (rb_obj_exivar_p(obj)) { + if (rb_obj_using_gen_fields_table_p(obj)) { rb_mark_generic_ivar(obj); } @@ -3123,7 +3420,8 @@ rb_gc_mark_children(void *objspace, VALUE obj) switch (BUILTIN_TYPE(obj)) { case T_CLASS: - if (FL_TEST_RAW(obj, FL_SINGLETON)) { + if (FL_TEST_RAW(obj, FL_SINGLETON) && + !rb_gc_checking_shareable()) { gc_mark_internal(RCLASS_ATTACHED_OBJECT(obj)); } // Continue to the shared T_CLASS/T_MODULE @@ -3131,12 +3429,18 @@ rb_gc_mark_children(void *objspace, VALUE obj) foreach_args.objspace = objspace; foreach_args.obj = obj; rb_class_classext_foreach(obj, gc_mark_classext_module, (void *)&foreach_args); + if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) { + gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object); + } break; case T_ICLASS: foreach_args.objspace = objspace; foreach_args.obj = obj; rb_class_classext_foreach(obj, gc_mark_classext_iclass, (void *)&foreach_args); + if (BOX_USER_P(RCLASS_PRIME_BOX(obj))) { + gc_mark_internal(RCLASS_PRIME_BOX(obj)->box_object); + } break; case T_ARRAY: @@ -3177,15 +3481,12 @@ rb_gc_mark_children(void *objspace, VALUE obj) break; case T_DATA: { - bool typed_data = RTYPEDDATA_P(obj); - void *const ptr = typed_data ? RTYPEDDATA_GET_DATA(obj) : DATA_PTR(obj); + void *const ptr = RTYPEDDATA_GET_DATA(obj); - if (typed_data) { - gc_mark_internal(RTYPEDDATA(obj)->fields_obj); - } + gc_mark_internal(RTYPEDDATA(obj)->fields_obj); if (ptr) { - if (typed_data && gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) { + if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) { size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj); for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) { @@ -3193,9 +3494,7 @@ rb_gc_mark_children(void *objspace, VALUE obj) } } else { - RUBY_DATA_FUNC mark_func = typed_data ? - RTYPEDDATA_TYPE(obj)->function.dmark : - RDATA(obj)->dmark; + RUBY_DATA_FUNC mark_func = RTYPEDDATA_TYPE(obj)->function.dmark; if (mark_func) (*mark_func)(ptr); } } @@ -3204,28 +3503,19 @@ rb_gc_mark_children(void *objspace, VALUE obj) } case T_OBJECT: { - if (rb_shape_obj_too_complex_p(obj)) { + uint32_t len; + if (rb_obj_shape_complex_p(obj)) { gc_mark_tbl_no_pin(ROBJECT_FIELDS_HASH(obj)); + len = ROBJECT_FIELDS_COUNT_COMPLEX(obj); } else { const VALUE * const ptr = ROBJECT_FIELDS(obj); - uint32_t len = ROBJECT_FIELDS_COUNT(obj); + len = ROBJECT_FIELDS_COUNT_NOT_COMPLEX(obj); for (uint32_t i = 0; i < len; i++) { gc_mark_internal(ptr[i]); } } - - attr_index_t fields_count = ROBJECT_FIELDS_COUNT(obj); - if (fields_count) { - VALUE klass = RBASIC_CLASS(obj); - - // Increment max_iv_count if applicable, used to determine size pool allocation - if (RCLASS_MAX_IV_COUNT(klass) < fields_count) { - RCLASS_SET_MAX_IV_COUNT(klass, fields_count); - } - } - break; } @@ -3272,7 +3562,7 @@ rb_gc_mark_children(void *objspace, VALUE obj) gc_mark_internal(ptr[i]); } - if (!FL_TEST_RAW(obj, RSTRUCT_GEN_FIELDS)) { + if (rb_obj_shape_has_fields(obj) && !FL_TEST_RAW(obj, RSTRUCT_GEN_FIELDS)) { gc_mark_internal(RSTRUCT_FIELDS_OBJ(obj)); } @@ -3294,21 +3584,48 @@ rb_gc_obj_optimal_size(VALUE obj) { switch (BUILTIN_TYPE(obj)) { case T_ARRAY: - return rb_ary_size_as_embedded(obj); + { + size_t size = rb_ary_size_as_embedded(obj); + if (rb_gc_size_allocatable_p(size)) { + return size; + } + else { + return sizeof(struct RArray); + } + } case T_OBJECT: - if (rb_shape_obj_too_complex_p(obj)) { + if (rb_obj_shape_complex_p(obj)) { return sizeof(struct RObject); } else { - return rb_obj_embedded_size(ROBJECT_FIELDS_CAPACITY(obj)); + size_t size = rb_obj_embedded_size(ROBJECT_FIELDS_CAPACITY(obj)); + if (rb_gc_size_allocatable_p(size)) { + return size; + } + else { + return sizeof(struct RObject); + } } case T_STRING: - return rb_str_size_as_embedded(obj); + { + size_t size = rb_str_size_as_embedded(obj); + if (rb_gc_size_allocatable_p(size)) { + return size; + } + else { + return sizeof(struct RString); + } + } case T_HASH: - return sizeof(struct RHash) + (RHASH_ST_TABLE_P(obj) ? sizeof(st_table) : sizeof(ar_table)); + { + if (RB_OBJ_FROZEN(obj) && RHASH_AR_TABLE_P(obj)) { + return sizeof(struct RHash) + offsetof(ar_table, pairs) + RHASH_AR_TABLE_BOUND(obj) * sizeof(ar_table_pair); + } + return sizeof(struct RHash) + (RHASH_ST_TABLE_P(obj) ? sizeof(st_table) : sizeof(ar_table)); + } default: return 0; @@ -3342,14 +3659,11 @@ rb_gc_copy_attributes(VALUE dest, VALUE obj) rb_gc_impl_copy_attributes(rb_gc_get_objspace(), dest, obj); } +#if USE_MODULAR_GC int rb_gc_modular_gc_loaded_p(void) { -#if USE_MODULAR_GC return rb_gc_functions.modular_gc_loaded_p; -#else - return false; -#endif } const char * @@ -3365,6 +3679,7 @@ rb_gc_active_gc_name(void) return gc_name; } +#endif struct rb_gc_object_metadata_entry * rb_gc_object_metadata(VALUE obj) @@ -3402,10 +3717,15 @@ rb_gc_register_address(VALUE *addr) VALUE obj = *addr; - struct global_object_list *tmp = ALLOC(struct global_object_list); - tmp->next = vm->global_object_list; - tmp->varptr = addr; - vm->global_object_list = tmp; + RB_VM_LOCKING() { + if (vm->global_object_list_size == vm->global_object_list_capa) { + size_t new_capa = vm->global_object_list_capa ? vm->global_object_list_capa * 2 : 64; + SIZED_REALLOC_N(vm->global_object_list, VALUE *, new_capa, vm->global_object_list_capa); + vm->global_object_list_capa = new_capa; + } + + vm->global_object_list[vm->global_object_list_size++] = addr; + } /* * Because some C extensions have assignment-then-register bugs, @@ -3423,22 +3743,20 @@ void rb_gc_unregister_address(VALUE *addr) { rb_vm_t *vm = GET_VM(); - struct global_object_list *tmp = vm->global_object_list; - - if (tmp->varptr == addr) { - vm->global_object_list = tmp->next; - xfree(tmp); - return; - } - while (tmp->next) { - if (tmp->next->varptr == addr) { - struct global_object_list *t = tmp->next; - - tmp->next = tmp->next->next; - xfree(t); - break; + RB_VM_LOCKING() { + size_t index; + for (index = 0; index < vm->global_object_list_size; index++) { + if (addr == vm->global_object_list[index]) { + MEMMOVE( + vm->global_object_list[index], + &vm->global_object_list[index + 1], + VALUE *, + vm->global_object_list_size - index - 1 + ); + vm->global_object_list_size--; + break; + } } - tmp = tmp->next; } } @@ -3539,19 +3857,21 @@ gc_ref_update_object(void *objspace, VALUE v) { VALUE *ptr = ROBJECT_FIELDS(v); - if (rb_shape_obj_too_complex_p(v)) { - gc_ref_update_table_values_only(ROBJECT_FIELDS_HASH(v)); - return; - } + if (FL_TEST_RAW(v, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(v)) { + gc_ref_update_table_values_only(ROBJECT_FIELDS_HASH(v)); + return; + } - size_t slot_size = rb_gc_obj_slot_size(v); - size_t embed_size = rb_obj_embedded_size(ROBJECT_FIELDS_CAPACITY(v)); - if (slot_size >= embed_size && !RB_FL_TEST_RAW(v, ROBJECT_EMBED)) { - // Object can be re-embedded - memcpy(ROBJECT(v)->as.ary, ptr, sizeof(VALUE) * ROBJECT_FIELDS_COUNT(v)); - RB_FL_SET_RAW(v, ROBJECT_EMBED); - xfree(ptr); - ptr = ROBJECT(v)->as.ary; + size_t slot_size = rb_gc_obj_slot_size(v); + size_t embed_size = rb_obj_embedded_size(ROBJECT_FIELDS_CAPACITY(v)); + if (slot_size >= embed_size) { + // Object can be re-embedded + memcpy(ROBJECT(v)->as.ary, ptr, sizeof(VALUE) * ROBJECT_FIELDS_COUNT(v)); + SIZED_FREE_N(ptr, ROBJECT_FIELDS_CAPACITY(v)); + FL_UNSET_RAW(v, ROBJECT_HEAP); + ptr = ROBJECT(v)->as.ary; + } } for (uint32_t i = 0; i < ROBJECT_FIELDS_COUNT(v); i++) { @@ -3572,6 +3892,36 @@ rb_gc_update_tbl_refs(st_table *ptr) gc_update_table_refs(ptr); } +static int +rb_gc_update_set_refs_i(st_data_t key, st_data_t value, st_data_t argp, int error) +{ + if (rb_gc_location((VALUE)key) != (VALUE)key) { + return ST_REPLACE; + } + + return ST_CONTINUE; +} + +static int +rb_gc_update_set_refs_replace_i(st_data_t *key, st_data_t *value, st_data_t argp, int existing) +{ + if (rb_gc_location((VALUE)*key) != (VALUE)*key) { + *key = rb_gc_location((VALUE)*key); + } + + return ST_CONTINUE; +} + +void +rb_gc_update_set_refs(st_table *tbl) +{ + if (!tbl || tbl->num_entries == 0) return; + + if (st_foreach_with_replace(tbl, rb_gc_update_set_refs_i, rb_gc_update_set_refs_replace_i, 0)) { + rb_raise(rb_eRuntimeError, "hash modified during iteration"); + } +} + static void gc_ref_update_hash(void *objspace, VALUE v) { @@ -3597,7 +3947,7 @@ check_id_table_move(VALUE value, void *data) { void *objspace = (void *)data; - if (rb_gc_impl_object_moved_p(objspace, (VALUE)value)) { + if (gc_object_moved_p_internal(objspace, (VALUE)value)) { return ID_TABLE_REPLACE; } @@ -3641,7 +3991,7 @@ update_id_table(VALUE *value, void *data, int existing) { void *objspace = (void *)data; - if (rb_gc_impl_object_moved_p(objspace, (VALUE)*value)) { + if (gc_object_moved_p_internal(objspace, (VALUE)*value)) { *value = gc_location_internal(objspace, (VALUE)*value); } @@ -3657,38 +4007,15 @@ update_m_tbl(void *objspace, struct rb_id_table *tbl) } static enum rb_id_table_iterator_result -update_cvc_tbl_i(VALUE cvc_entry, void *objspace) -{ - struct rb_cvar_class_tbl_entry *entry; - - entry = (struct rb_cvar_class_tbl_entry *)cvc_entry; - - if (entry->cref) { - TYPED_UPDATE_IF_MOVED(objspace, rb_cref_t *, entry->cref); - } - - entry->class_value = gc_location_internal(objspace, entry->class_value); - - return ID_TABLE_CONTINUE; -} - -static void -update_cvc_tbl(void *objspace, struct rb_id_table *tbl) -{ - if (!tbl) return; - rb_id_table_foreach_values(tbl, update_cvc_tbl_i, objspace); -} - -static enum rb_id_table_iterator_result update_const_tbl_i(VALUE value, void *objspace) { rb_const_entry_t *ce = (rb_const_entry_t *)value; - if (rb_gc_impl_object_moved_p(objspace, ce->value)) { + if (gc_object_moved_p_internal(objspace, ce->value)) { ce->value = gc_location_internal(objspace, ce->value); } - if (rb_gc_impl_object_moved_p(objspace, ce->file)) { + if (gc_object_moved_p_internal(objspace, ce->file)) { ce->file = gc_location_internal(objspace, ce->file); } @@ -3703,20 +4030,6 @@ update_const_tbl(void *objspace, struct rb_id_table *tbl) } static void -update_subclasses(void *objspace, rb_classext_t *ext) -{ - rb_subclass_entry_t *entry; - rb_subclass_anchor_t *anchor = RCLASSEXT_SUBCLASSES(ext); - if (!anchor) return; - entry = anchor->head; - while (entry) { - if (entry->klass) - UPDATE_IF_MOVED(objspace, entry->klass); - entry = entry->next; - } -} - -static void update_superclasses(rb_objspace_t *objspace, rb_classext_t *ext) { if (RCLASSEXT_SUPERCLASSES_WITH_SELF(ext)) { @@ -3739,7 +4052,7 @@ update_classext_values(rb_objspace_t *objspace, rb_classext_t *ext, bool is_icla } static void -update_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg) +update_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) { struct classext_foreach_args *args = (struct classext_foreach_args *)arg; rb_objspace_t *objspace = args->objspace; @@ -3755,15 +4068,17 @@ update_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg) update_const_tbl(objspace, RCLASSEXT_CONST_TBL(ext)); } UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext)); - update_cvc_tbl(objspace, RCLASSEXT_CVC_TBL(ext)); + UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext)); update_superclasses(objspace, ext); - update_subclasses(objspace, ext); + if (RCLASSEXT_SUBCLASSES(ext)) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext)); + } update_classext_values(objspace, ext, false); } static void -update_iclass_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void *arg) +update_iclass_classext(rb_classext_t *ext, bool is_prime, VALUE box_value, void *arg) { struct classext_foreach_args *args = (struct classext_foreach_args *)arg; rb_objspace_t *objspace = args->objspace; @@ -3774,7 +4089,10 @@ update_iclass_classext(rb_classext_t *ext, bool is_prime, VALUE namespace, void update_m_tbl(objspace, RCLASSEXT_M_TBL(ext)); update_m_tbl(objspace, RCLASSEXT_CALLABLE_M_TBL(ext)); UPDATE_IF_MOVED(objspace, RCLASSEXT_CC_TBL(ext)); - update_subclasses(objspace, ext); + UPDATE_IF_MOVED(objspace, RCLASSEXT_CVC_TBL(ext)); + if (RCLASSEXT_SUBCLASSES(ext)) { + UPDATE_IF_MOVED(objspace, RCLASSEXT_SUBCLASSES(ext)); + } update_classext_values(objspace, ext, true); } @@ -3819,23 +4137,6 @@ vm_weak_table_foreach_update_weak_key(st_data_t *key, st_data_t *value, st_data_ } static int -vm_weak_table_cc_refinement_foreach(st_data_t key, st_data_t data, int error) -{ - struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; - - return iter_data->callback((VALUE)key, iter_data->data); -} - -static int -vm_weak_table_cc_refinement_foreach_update_update(st_data_t *key, st_data_t data, int existing) -{ - struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; - - return iter_data->update_callback((VALUE *)key, iter_data->data); -} - - -static int vm_weak_table_sym_set_foreach(VALUE *sym_ptr, void *data) { VALUE sym = *sym_ptr; @@ -3896,7 +4197,11 @@ vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data) break; case ST_DELETE: - RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID); + // When we're removing an object from the weak ref table, we need to + // set the shape on it so that the GC finalizer won't try to remove + // it again. A "root shape" indicates to the GC that this object + // has no fields on it, hence it won't be in the gen fields table. + RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER); return ST_DELETE; case ST_REPLACE: { @@ -3974,25 +4279,21 @@ rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, switch (table) { case RB_GC_VM_CI_TABLE: { - if (vm->ci_table) { - st_foreach_with_replace( - vm->ci_table, - vm_weak_table_foreach_weak_key, - vm_weak_table_foreach_update_weak_key, - (st_data_t)&foreach_data - ); - } + st_foreach_with_replace( + &vm->ci_table, + vm_weak_table_foreach_weak_key, + vm_weak_table_foreach_update_weak_key, + (st_data_t)&foreach_data + ); break; } case RB_GC_VM_OVERLOADED_CME_TABLE: { - if (vm->overloaded_cme_table) { - st_foreach_with_replace( - vm->overloaded_cme_table, - vm_weak_table_foreach_weak_key, - vm_weak_table_foreach_update_weak_key, - (st_data_t)&foreach_data - ); - } + st_foreach_with_replace( + &vm->overloaded_cme_table, + vm_weak_table_foreach_weak_key, + vm_weak_table_foreach_update_weak_key, + (st_data_t)&foreach_data + ); break; } case RB_GC_VM_GLOBAL_SYMBOLS_TABLE: { @@ -4031,17 +4332,6 @@ rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, ); break; } - case RB_GC_VM_CC_REFINEMENT_TABLE: { - if (vm->cc_refinement_table) { - set_foreach_with_replace( - vm->cc_refinement_table, - vm_weak_table_cc_refinement_foreach, - vm_weak_table_cc_refinement_foreach_update_update, - (st_data_t)&foreach_data - ); - } - break; - } case RB_GC_VM_WEAK_TABLE_COUNT: rb_bug("Unreachable"); default: @@ -4066,6 +4356,14 @@ rb_gc_update_vm_references(void *objspace) rb_yjit_root_update_references(); } #endif + +#if USE_ZJIT + void rb_zjit_root_update_references(void); // in Rust + + if (rb_zjit_enabled_p) { + rb_zjit_root_update_references(); + } +#endif } void @@ -4130,15 +4428,12 @@ rb_gc_update_object_references(void *objspace, VALUE obj) case T_DATA: /* Call the compaction callback, if it exists */ { - bool typed_data = RTYPEDDATA_P(obj); - void *const ptr = typed_data ? RTYPEDDATA_GET_DATA(obj) : DATA_PTR(obj); + void *const ptr = RTYPEDDATA_GET_DATA(obj); - if (typed_data) { - UPDATE_IF_MOVED(objspace, RTYPEDDATA(obj)->fields_obj); - } + UPDATE_IF_MOVED(objspace, RTYPEDDATA(obj)->fields_obj); if (ptr) { - if (typed_data && gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) { + if (gc_declarative_marking_p(RTYPEDDATA_TYPE(obj))) { size_t *offset_list = TYPED_DATA_REFS_OFFSET_LIST(obj); for (size_t offset = *offset_list; offset != RUBY_REF_END; offset = *offset_list++) { @@ -4146,7 +4441,7 @@ rb_gc_update_object_references(void *objspace, VALUE obj) *ref = gc_location_internal(objspace, *ref); } } - else if (typed_data) { + else { RUBY_DATA_FUNC compact_func = RTYPEDDATA_TYPE(obj)->function.dcompact; if (compact_func) (*compact_func)(ptr); } @@ -4575,7 +4870,7 @@ rb_method_type_name(rb_method_type_t type) { switch (type) { case VM_METHOD_TYPE_ISEQ: return "iseq"; - case VM_METHOD_TYPE_ATTRSET: return "attrest"; + case VM_METHOD_TYPE_ATTRSET: return "attrset"; case VM_METHOD_TYPE_IVAR: return "ivar"; case VM_METHOD_TYPE_BMETHOD: return "bmethod"; case VM_METHOD_TYPE_ALIAS: return "alias"; @@ -4668,7 +4963,7 @@ rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj APPEND_S("(temporary internal)"); } else if (RTEST(RBASIC(obj)->klass)) { - VALUE class_path = rb_class_path_cached(RBASIC(obj)->klass); + VALUE class_path = rb_mod_name(RBASIC(obj)->klass); if (!NIL_P(class_path)) { APPEND_F("%s ", RSTRING_PTR(class_path)); } @@ -4715,6 +5010,10 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU } break; case T_STRING: { + APPEND_F("[%s%s] ", + C(FL_TEST(obj, RSTRING_FSTR), "F"), + C(RB_OBJ_FROZEN(obj), "R")); + if (STR_SHARED_P(obj)) { APPEND_F(" [shared] len: %ld", RSTRING_LEN(obj)); } @@ -4750,7 +5049,7 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU case T_CLASS: case T_MODULE: { - VALUE class_path = rb_class_path_cached(obj); + VALUE class_path = rb_mod_name(obj); if (!NIL_P(class_path)) { APPEND_F("%s", RSTRING_PTR(class_path)); } @@ -4761,7 +5060,7 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU } case T_ICLASS: { - VALUE class_path = rb_class_path_cached(RBASIC_CLASS(obj)); + VALUE class_path = rb_mod_name(RBASIC_CLASS(obj)); if (!NIL_P(class_path)) { APPEND_F("src:%s", RSTRING_PTR(class_path)); } @@ -4769,21 +5068,18 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU } case T_OBJECT: { - if (rb_shape_obj_too_complex_p(obj)) { - size_t hash_len = rb_st_table_size(ROBJECT_FIELDS_HASH(obj)); - APPEND_F("(too_complex) len:%zu", hash_len); - } - else { - uint32_t len = ROBJECT_FIELDS_CAPACITY(obj); - - if (RBASIC(obj)->flags & ROBJECT_EMBED) { - APPEND_F("(embed) len:%d", len); + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { + if (rb_obj_shape_complex_p(obj)) { + size_t hash_len = rb_st_table_size(ROBJECT_FIELDS_HASH(obj)); + APPEND_F("(complex) len:%zu", hash_len); } else { - VALUE *ptr = ROBJECT_FIELDS(obj); - APPEND_F("len:%d ptr:%p", len, (void *)ptr); + APPEND_F("(embed) len:%d capa:%d", RSHAPE_LEN(RBASIC_SHAPE_ID(obj)), ROBJECT_FIELDS_CAPACITY(obj)); } } + else { + APPEND_F("len:%d capa:%d ptr:%p", RSHAPE_LEN(RBASIC_SHAPE_ID(obj)), ROBJECT_FIELDS_CAPACITY(obj), (void *)ROBJECT_FIELDS(obj)); + } } break; case T_DATA: { @@ -4801,12 +5097,6 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU APPEND_F("r:%d", r->pub.id); } } - else { - const char * const type_name = rb_objspace_data_type_name(obj); - if (type_name) { - APPEND_F("%s", type_name); - } - } break; } case T_IMEMO: { @@ -4861,7 +5151,7 @@ rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALU case imemo_callcache: { const struct rb_callcache *cc = (const struct rb_callcache *)obj; - VALUE class_path = vm_cc_valid(cc) ? rb_class_path_cached(cc->klass) : Qnil; + VALUE class_path = vm_cc_valid(cc) ? rb_mod_name(cc->klass) : Qnil; const rb_callable_method_entry_t *cme = vm_cc_cme(cc); APPEND_F("(klass:%s cme:%s%s (%p) call:%p", @@ -4946,12 +5236,6 @@ rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj) #undef APPEND_F #undef BUFF_ARGS -#if RGENGC_OBJ_INFO -#define OBJ_INFO_BUFFERS_NUM 10 -#define OBJ_INFO_BUFFERS_SIZE 0x100 -static rb_atomic_t obj_info_buffers_index = 0; -static char obj_info_buffers[OBJ_INFO_BUFFERS_NUM][OBJ_INFO_BUFFERS_SIZE]; - /* Increments *var atomically and resets *var to 0 when maxval is * reached. Returns the wraparound old *var value (0...maxval). */ static rb_atomic_t @@ -4969,17 +5253,18 @@ atomic_inc_wraparound(rb_atomic_t *var, const rb_atomic_t maxval) static const char * obj_info(VALUE obj) { - rb_atomic_t index = atomic_inc_wraparound(&obj_info_buffers_index, OBJ_INFO_BUFFERS_NUM); - char *const buff = obj_info_buffers[index]; - return rb_raw_obj_info(buff, OBJ_INFO_BUFFERS_SIZE, obj); -} -#else -static const char * -obj_info(VALUE obj) -{ + if (RGENGC_OBJ_INFO) { + static struct { + rb_atomic_t index; + char buffers[10][0x100]; + } info = {0}; + + rb_atomic_t index = atomic_inc_wraparound(&info.index, numberof(info.buffers)); + char *const buff = info.buffers[index]; + return rb_raw_obj_info(buff, sizeof(info.buffers[0]), obj); + } return obj_type_name(obj); } -#endif /* ------------------------ Extended allocator ------------------------ @@ -5008,11 +5293,7 @@ gc_raise(VALUE exc, const char *fmt, ...) exc, fmt, &ap, }; - if (ruby_thread_has_gvl_p()) { - gc_vraise(&argv); - UNREACHABLE; - } - else if (ruby_native_thread_p()) { + if (ruby_native_thread_p()) { rb_thread_call_with_gvl(gc_vraise, &argv); UNREACHABLE; } @@ -5118,6 +5399,14 @@ ruby_xmalloc(size_t size) return handle_malloc_failure(ruby_xmalloc_body(size)); } +static bool +malloc_gc_allowed(void) +{ + rb_ractor_t *r = rb_current_ractor_raw(false); + + return r == NULL || !r->malloc_gc_disabled; +} + static void * ruby_xmalloc_body(size_t size) { @@ -5125,7 +5414,7 @@ ruby_xmalloc_body(size_t size) negative_size_allocation_error("too large allocation size"); } - return rb_gc_impl_malloc(rb_gc_get_objspace(), size); + return rb_gc_impl_malloc(rb_gc_get_objspace(), size, malloc_gc_allowed()); } void @@ -5155,7 +5444,7 @@ ruby_xmalloc2(size_t n, size_t size) static void * ruby_xmalloc2_body(size_t n, size_t size) { - return rb_gc_impl_malloc(rb_gc_get_objspace(), xmalloc2_size(n, size)); + return rb_gc_impl_malloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed()); } static void *ruby_xcalloc_body(size_t n, size_t size); @@ -5169,65 +5458,65 @@ ruby_xcalloc(size_t n, size_t size) static void * ruby_xcalloc_body(size_t n, size_t size) { - return rb_gc_impl_calloc(rb_gc_get_objspace(), xmalloc2_size(n, size)); + return rb_gc_impl_calloc(rb_gc_get_objspace(), xmalloc2_size(n, size), malloc_gc_allowed()); } -static void *ruby_sized_xrealloc_body(void *ptr, size_t new_size, size_t old_size); +static void *ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size); -#ifdef ruby_sized_xrealloc -#undef ruby_sized_xrealloc +#ifdef ruby_xrealloc_sized +#undef ruby_xrealloc_sized #endif void * -ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) +ruby_xrealloc_sized(void *ptr, size_t new_size, size_t old_size) { - return handle_malloc_failure(ruby_sized_xrealloc_body(ptr, new_size, old_size)); + return handle_malloc_failure(ruby_xrealloc_sized_body(ptr, new_size, old_size)); } static void * -ruby_sized_xrealloc_body(void *ptr, size_t new_size, size_t old_size) +ruby_xrealloc_sized_body(void *ptr, size_t new_size, size_t old_size) { if ((ssize_t)new_size < 0) { negative_size_allocation_error("too large allocation size"); } - return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, new_size, old_size); + return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, new_size, old_size, malloc_gc_allowed()); } void * ruby_xrealloc(void *ptr, size_t new_size) { - return ruby_sized_xrealloc(ptr, new_size, 0); + return ruby_xrealloc_sized(ptr, new_size, 0); } -static void *ruby_sized_xrealloc2_body(void *ptr, size_t n, size_t size, size_t old_n); +static void *ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n); -#ifdef ruby_sized_xrealloc2 -#undef ruby_sized_xrealloc2 +#ifdef ruby_xrealloc2_sized +#undef ruby_xrealloc2_sized #endif void * -ruby_sized_xrealloc2(void *ptr, size_t n, size_t size, size_t old_n) +ruby_xrealloc2_sized(void *ptr, size_t n, size_t size, size_t old_n) { - return handle_malloc_failure(ruby_sized_xrealloc2_body(ptr, n, size, old_n)); + return handle_malloc_failure(ruby_xrealloc2_sized_body(ptr, n, size, old_n)); } static void * -ruby_sized_xrealloc2_body(void *ptr, size_t n, size_t size, size_t old_n) +ruby_xrealloc2_sized_body(void *ptr, size_t n, size_t size, size_t old_n) { size_t len = xmalloc2_size(n, size); - return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, len, old_n * size); + return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, len, old_n * size, malloc_gc_allowed()); } void * ruby_xrealloc2(void *ptr, size_t n, size_t size) { - return ruby_sized_xrealloc2(ptr, n, size, 0); + return ruby_xrealloc2_sized(ptr, n, size, 0); } -#ifdef ruby_sized_xfree -#undef ruby_sized_xfree +#ifdef ruby_xfree_sized +#undef ruby_xfree_sized #endif void -ruby_sized_xfree(void *x, size_t size) +ruby_xfree_sized(void *x, size_t size) { if (LIKELY(x)) { /* It's possible for a C extension's pthread destructor function set by pthread_key_create @@ -5245,7 +5534,7 @@ ruby_sized_xfree(void *x, size_t size) void ruby_xfree(void *x) { - ruby_sized_xfree(x, 0); + ruby_xfree_sized(x, 0); } void * @@ -5314,11 +5603,11 @@ ruby_mimcalloc(size_t num, size_t size) { void *mem; #if CALC_EXACT_MALLOC_SIZE - struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(num, size); - if (UNLIKELY(t.left)) { + struct rbimpl_size_overflow_tag t = rbimpl_size_mul_overflow(num, size); + if (UNLIKELY(t.overflowed)) { return NULL; } - size = t.right + sizeof(struct malloc_obj_info); + size = t.result + sizeof(struct malloc_obj_info); mem = calloc1(size); if (!mem) { return NULL; @@ -5386,6 +5675,80 @@ rb_gc_after_fork(rb_pid_t pid) rb_gc_impl_after_fork(rb_gc_get_objspace(), pid); } +bool +rb_gc_obj_shareable_p(VALUE obj) +{ + return RB_OBJ_SHAREABLE_P(obj); +} + +void +rb_gc_rp(VALUE obj) +{ + rp(obj); +} + +struct check_shareable_data { + VALUE parent; + long err_count; +}; + +static void +check_shareable_i(const VALUE child, void *ptr) +{ + struct check_shareable_data *data = (struct check_shareable_data *)ptr; + + if (!rb_gc_obj_shareable_p(child)) { + fprintf(stderr, "(a) "); + rb_gc_rp(data->parent); + fprintf(stderr, "(b) "); + rb_gc_rp(child); + fprintf(stderr, "check_shareable_i: shareable (a) -> unshareable (b)\n"); + + data->err_count++; + rb_bug("!! violate shareable constraint !!"); + } +} + +static bool gc_checking_shareable = false; + +static void +gc_verify_shareable(void *objspace, VALUE obj, void *data) +{ + // while gc_checking_shareable is true, + // other Ractors should not run the GC, until the flag is not local. + // TODO: remove VM locking if the flag is Ractor local + + unsigned int lev = RB_GC_VM_LOCK(); + { + gc_checking_shareable = true; + rb_objspace_reachable_objects_from(obj, check_shareable_i, (void *)data); + gc_checking_shareable = false; + } + RB_GC_VM_UNLOCK(lev); +} + +// TODO: only one level (non-recursive) +void +rb_gc_verify_shareable(VALUE obj) +{ + rb_objspace_t *objspace = rb_gc_get_objspace(); + struct check_shareable_data data = { + .parent = obj, + .err_count = 0, + }; + gc_verify_shareable(objspace, obj, &data); + + if (data.err_count > 0) { + rb_bug("rb_gc_verify_shareable"); + } +} + +bool +rb_gc_checking_shareable(void) +{ + return gc_checking_shareable; +} + /* * Document-module: ObjectSpace * |