summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAaron Patterson <tenderlove@ruby-lang.org>2020-11-02 14:40:29 -0800
committerAaron Patterson <tenderlove@ruby-lang.org>2020-11-02 14:42:48 -0800
commit67b2c21c327c96d80b8a0fe02a96d417e85293e8 (patch)
treec4ea1236016fc2b7e40582a69966d35c5f8c7289
parent79b242260bc0530503dde85eda8e79b1c2aa9a6e (diff)
Add `GC.auto_compact= true/false` and `GC.auto_compact`
* `GC.auto_compact=`, `GC.auto_compact` can be used to control when compaction runs. Setting `auto_compact=` to true will cause compaction to occurr duing major collections. At the moment, compaction adds significant overhead to major collections, so please test first! [Feature #17176]
-rw-r--r--NEWS.md10
-rw-r--r--compile.c3
-rw-r--r--gc.c993
-rw-r--r--gc.rb56
-rw-r--r--internal/vm.h1
-rw-r--r--proc.c5
-rw-r--r--test/ruby/test_gc_compact.rb65
-rw-r--r--transient_heap.c17
-rw-r--r--vm.c30
9 files changed, 807 insertions, 373 deletions
diff --git a/NEWS.md b/NEWS.md
index 2fa1b3db2e..ecf9c77aaa 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -262,6 +262,16 @@ Outstanding ones only.
* Warning#warn now supports a category kwarg.
[[Feature #17122]]
+* GC
+ * New method
+
+ * `GC.auto_compact=`, `GC.auto_compact` can be used to control when
+ compaction runs. Setting `auto_compact=` to true will cause
+ compaction to occurr duing major collections. At the moment,
+ compaction adds significant overhead to major collections, so please
+ test first!
+ [[Feature #17176]]
+
## Stdlib updates
Outstanding ones only.
diff --git a/compile.c b/compile.c
index bf2c4f88a6..f6f5a1d848 100644
--- a/compile.c
+++ b/compile.c
@@ -1304,6 +1304,9 @@ new_insn_send(rb_iseq_t *iseq, int line_no, ID id, VALUE argc, const rb_iseq_t *
VALUE *operands = compile_data_calloc2(iseq, sizeof(VALUE), 2);
operands[0] = (VALUE)new_callinfo(iseq, id, FIX2INT(argc), FIX2INT(flag), keywords, blockiseq != NULL);
operands[1] = (VALUE)blockiseq;
+ if (blockiseq) {
+ RB_OBJ_WRITTEN(iseq, Qundef, blockiseq);
+ }
return new_insn_core(iseq, line_no, BIN(send), 2, operands);
}
diff --git a/gc.c b/gc.c
index af7fcef48c..65876894bd 100644
--- a/gc.c
+++ b/gc.c
@@ -19,6 +19,15 @@
# include "ruby/ruby.h"
#endif
+#include <signal.h>
+
+#define sighandler_t ruby_sighandler_t
+
+#ifndef _WIN32
+#include <unistd.h>
+#include <sys/mman.h>
+#endif
+
#include <setjmp.h>
#include <stdarg.h>
#include <stdio.h>
@@ -482,6 +491,7 @@ typedef enum {
GPR_FLAG_HAVE_FINALIZE = 0x4000,
GPR_FLAG_IMMEDIATE_MARK = 0x8000,
GPR_FLAG_FULL_MARK = 0x10000,
+ GPR_FLAG_COMPACT = 0x20000,
GPR_DEFAULT_REASON =
(GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
@@ -497,6 +507,7 @@ typedef struct gc_profile_record {
size_t heap_total_objects;
size_t heap_use_size;
size_t heap_total_size;
+ size_t moved_objects;
#if GC_PROFILE_MORE_DETAIL
double gc_mark_time;
@@ -529,13 +540,12 @@ typedef struct gc_profile_record {
#endif
} gc_profile_record;
+#define FL_FROM_FREELIST FL_USER0
+
struct RMoved {
VALUE flags;
+ VALUE dummy;
VALUE destination;
- union {
- struct list_node node;
- struct list_head head;
- } as;
};
#define RMOVED(obj) ((struct RMoved *)(obj))
@@ -641,6 +651,8 @@ typedef struct rb_heap_struct {
struct heap_page *using_page;
struct list_head pages;
struct heap_page *sweeping_page; /* iterator for .pages */
+ struct heap_page *compact_cursor;
+ size_t compact_cursor_index;
#if GC_ENABLE_INCREMENTAL_MARK
struct heap_page *pooled_pages;
#endif
@@ -724,6 +736,7 @@ typedef struct rb_objspace {
size_t minor_gc_count;
size_t major_gc_count;
size_t compact_count;
+ size_t read_barrier_faults;
#if RGENGC_PROFILE > 0
size_t total_generated_normal_object_count;
size_t total_generated_shady_object_count;
@@ -780,6 +793,7 @@ typedef struct rb_objspace {
struct {
size_t considered_count_table[T_MASK];
size_t moved_count_table[T_MASK];
+ size_t total_moved;
} rcompactor;
#if GC_ENABLE_INCREMENTAL_MARK
@@ -969,6 +983,7 @@ int ruby_gc_debug_indent = 0;
#endif
VALUE rb_mGC;
int ruby_disable_gc = 0;
+int ruby_enable_autocompact = 0;
void rb_iseq_mark(const rb_iseq_t *iseq);
void rb_iseq_update_references(rb_iseq_t *iseq);
@@ -3801,6 +3816,7 @@ is_live_object(rb_objspace_t *objspace, VALUE ptr)
{
switch (BUILTIN_TYPE(ptr)) {
case T_NONE:
+ case T_MOVED:
case T_ZOMBIE:
return FALSE;
default:
@@ -3948,8 +3964,10 @@ cached_object_id(VALUE obj)
id = objspace->next_object_id;
objspace->next_object_id = rb_int_plus(id, INT2FIX(OBJ_ID_INCREMENT));
+ VALUE already_disabled = rb_gc_disable_no_rest();
st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
+ if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
FL_SET(obj, FL_SEEN_OBJ_ID);
return id;
@@ -4361,16 +4379,370 @@ gc_setup_mark_bits(struct heap_page *page)
memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
}
+static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
+static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free);
+
+static void
+lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
+{
+#if defined(_WIN32)
+ DWORD old_protect;
+
+ if (!VirtualProtect(body, HEAP_PAGE_SIZE, PAGE_NOACCESS, &old_protect)) {
+#else
+ if(mprotect(body, HEAP_PAGE_SIZE, PROT_NONE)) {
+#endif
+ rb_bug("Couldn't protect page %p", (void *)body);
+ } else {
+ gc_report(5, objspace, "Protecting page in move %p\n", (void *)body);
+ }
+}
+
+static void
+unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
+{
+#if defined(_WIN32)
+ DWORD old_protect;
+
+ if (!VirtualProtect(body, HEAP_PAGE_SIZE, PAGE_READWRITE, &old_protect)) {
+#else
+ if(mprotect(body, HEAP_PAGE_SIZE, PROT_READ | PROT_WRITE)) {
+#endif
+ rb_bug("Couldn't unprotect page %p", (void *)body);
+ } else {
+ gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body);
+ }
+}
+
+static short
+try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page, VALUE dest)
+{
+ struct heap_page * cursor = heap->compact_cursor;
+ char from_freelist = 0;
+
+ GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
+
+ /* T_NONE objects came from the free list. If the object is *not* a
+ * T_NONE, it is an object that just got freed but hasn't been
+ * added to the freelist yet */
+
+ if (BUILTIN_TYPE(dest) == T_NONE) {
+ from_freelist = 1;
+ }
+
+ while(1) {
+ size_t index = heap->compact_cursor_index;
+
+ bits_t *mark_bits = cursor->mark_bits;
+ bits_t *pin_bits = cursor->pinned_bits;
+ RVALUE * p = cursor->start;
+ RVALUE * offset = p - NUM_IN_PAGE(p);
+
+ /* Find an object to move and move it. Movable objects must be
+ * marked, so we iterate using the marking bitmap */
+ for (size_t i = index; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
+ bits_t bits = mark_bits[i] & ~pin_bits[i];
+
+ if (bits) {
+ p = offset + i * BITS_BITLENGTH;
+
+ do {
+ if (bits & 1) {
+ /* We're trying to move "p" */
+ objspace->rcompactor.considered_count_table[BUILTIN_TYPE((VALUE)p)]++;
+
+ if (gc_is_moveable_obj(objspace, (VALUE)p)) {
+ /* We were able to move "p" */
+ objspace->rcompactor.moved_count_table[BUILTIN_TYPE((VALUE)p)]++;
+ objspace->rcompactor.total_moved++;
+ gc_move(objspace, (VALUE)p, dest);
+ gc_pin(objspace, (VALUE)p);
+ heap->compact_cursor_index = i;
+ if (from_freelist) {
+ FL_SET((VALUE)p, FL_FROM_FREELIST);
+ }
+
+ return 1;
+ }
+ }
+ p++;
+ bits >>= 1;
+ } while (bits);
+ }
+ }
+
+ /* We couldn't find a movable object on the compact cursor, so lets
+ * move to the next page (previous page since we are traveling in the
+ * opposite direction of the sweep cursor) and look there. */
+
+ struct heap_page * next;
+
+ next = list_prev(&heap->pages, cursor, page_node);
+
+ /* Protect the current cursor since it probably has T_MOVED slots. */
+ lock_page_body(objspace, GET_PAGE_BODY(cursor->start));
+
+ heap->compact_cursor = next;
+ heap->compact_cursor_index = 0;
+ cursor = next;
+
+ // Cursors have met, lets quit. We set `heap->compact_cursor` equal
+ // to `heap->sweeping_page` so we know how far to iterate through
+ // the heap when unprotecting pages.
+ if (next == sweep_page) {
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static void
+gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap)
+{
+ struct heap_page *cursor = heap->compact_cursor;
+
+ while(cursor) {
+ unlock_page_body(objspace, GET_PAGE_BODY(cursor->start));
+ cursor = list_next(&heap->pages, cursor, page_node);
+ }
+}
+
+static void gc_update_references(rb_objspace_t * objspace, rb_heap_t *heap);
+static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page);
+
+static void read_barrier_handler(intptr_t address)
+{
+ VALUE obj;
+ rb_objspace_t * objspace = &rb_objspace;
+
+ address -= address % sizeof(RVALUE);
+
+ obj = (VALUE)address;
+
+ RB_VM_LOCK_ENTER();
+ {
+ unlock_page_body(objspace, GET_PAGE_BODY(obj));
+
+ objspace->profile.read_barrier_faults++;
+
+ invalidate_moved_page(objspace, GET_HEAP_PAGE(obj));
+ }
+ RB_VM_LOCK_LEAVE();
+}
+
+#if defined(_WIN32)
+LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
+typedef void (*signal_handler)(int);
+static signal_handler old_sigsegv_handler;
+
+static LONG WINAPI read_barrier_signal(EXCEPTION_POINTERS * info)
+{
+ /* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */
+ if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+ /* > The second array element specifies the virtual address of the inaccessible data.
+ * https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
+ *
+ * Use this address to invalidate the page */
+ read_barrier_handler((intptr_t)info->ExceptionRecord->ExceptionInformation[1]);
+ return EXCEPTION_CONTINUE_EXECUTION;
+ } else {
+ return EXCEPTION_CONTINUE_SEARCH;
+ }
+}
+
+static void uninstall_handlers(void)
+{
+ signal(SIGSEGV, old_sigsegv_handler);
+ SetUnhandledExceptionFilter(old_handler);
+}
+
+static void install_handlers(void)
+{
+ /* Remove SEGV handler so that the Unhandled Exception Filter handles it */
+ old_sigsegv_handler = signal(SIGSEGV, NULL);
+ /* Unhandled Exception Filter has access to the violation address similar
+ * to si_addr from sigaction */
+ old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
+}
+#else
+static struct sigaction old_sigbus_handler;
+static struct sigaction old_sigsegv_handler;
+
+static void
+read_barrier_signal(int sig, siginfo_t * info, void * data)
+{
+ read_barrier_handler((intptr_t)info->si_addr);
+}
+
+static void uninstall_handlers(void)
+{
+ sigaction(SIGBUS, &old_sigbus_handler, NULL);
+ sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
+}
+
+static void install_handlers(void)
+{
+ struct sigaction action;
+ memset(&action, 0, sizeof(struct sigaction));
+ sigemptyset(&action.sa_mask);
+ action.sa_sigaction = read_barrier_signal;
+ action.sa_flags = SA_SIGINFO | SA_ONSTACK;
+
+ sigaction(SIGBUS, &action, &old_sigbus_handler);
+ sigaction(SIGSEGV, &action, &old_sigsegv_handler);
+}
+#endif
+
+static void
+revert_stack_objects(VALUE stack_obj, void *ctx)
+{
+ rb_objspace_t * objspace = (rb_objspace_t*)ctx;
+
+ if (BUILTIN_TYPE(stack_obj) == T_MOVED) {
+ /* For now we'll revert the whole page if the object made it to the
+ * stack. I think we can change this to move just the one object
+ * back though */
+ invalidate_moved_page(objspace, GET_HEAP_PAGE(stack_obj));
+ }
+}
+
+static void
+check_stack_for_moved(rb_objspace_t *objspace)
+{
+ rb_execution_context_t *ec = GET_EC();
+ rb_vm_t *vm = rb_ec_vm_ptr(ec);
+ rb_vm_each_stack_value(vm, revert_stack_objects, (void*)objspace);
+}
+
+static void
+gc_compact_finish(rb_objspace_t *objspace, rb_heap_t *heap)
+{
+ GC_ASSERT(heap->sweeping_page == heap->compact_cursor);
+
+ gc_unprotect_pages(objspace, heap);
+ uninstall_handlers();
+
+ /* The mutator is allowed to run during incremental sweeping. T_MOVED
+ * objects can get pushed on the stack and when the compaction process
+ * finishes up, it may remove the read barrier before anything has a
+ * chance to read from the T_MOVED address. To fix this, we scan the stack
+ * then revert any moved objects that made it to the stack. */
+ check_stack_for_moved(objspace);
+
+ gc_update_references(objspace, heap);
+ heap->compact_cursor = NULL;
+ heap->compact_cursor_index = 0;
+ objspace->profile.compact_count++;
+ if (gc_prof_enabled(objspace)) {
+ gc_profile_record *record = gc_prof_record(objspace);
+ record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects;
+ }
+ rb_clear_constant_cache();
+ objspace->flags.during_compacting = FALSE;
+}
+
+static int
+gc_fill_swept_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page, int *freed_slots, int *empty_slots)
+{
+ /* Find any pinned but not marked objects and try to fill those slots */
+ int i;
+ int moved_slots = 0;
+ int finished_compacting = 0;
+ bits_t *mark_bits, *pin_bits;
+ bits_t bitset;
+ RVALUE *p, *offset;
+
+ mark_bits = sweep_page->mark_bits;
+ pin_bits = sweep_page->pinned_bits;
+
+ p = sweep_page->start;
+ offset = p - NUM_IN_PAGE(p);
+
+ struct heap_page * cursor = heap->compact_cursor;
+
+ unlock_page_body(objspace, GET_PAGE_BODY(cursor->start));
+
+ for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
+ /* *Want to move* objects are pinned but not marked. */
+ bitset = pin_bits[i] & ~mark_bits[i];
+
+ if (bitset) {
+ p = offset + i * BITS_BITLENGTH;
+ do {
+ if (bitset & 1) {
+ VALUE dest = (VALUE)p;
+
+ GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(dest), dest));
+ GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(dest), dest));
+
+ CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(dest), dest);
+
+ if (finished_compacting) {
+ if (BUILTIN_TYPE(dest) == T_NONE) {
+ (*empty_slots)++;
+ } else {
+ (*freed_slots)++;
+ }
+ (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)dest, sizeof(RVALUE));
+ heap_page_add_freeobj(objspace, sweep_page, dest);
+ } else {
+ /* Zombie slots don't get marked, but we can't reuse
+ * their memory until they have their finalizers run.*/
+ if (BUILTIN_TYPE(dest) != T_ZOMBIE) {
+ if(!try_move(objspace, heap, sweep_page, dest)) {
+ finished_compacting = 1;
+ (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
+ gc_report(5, objspace, "Quit compacting, couldn't find an object to move\n");
+ if (BUILTIN_TYPE(dest) == T_NONE) {
+ (*empty_slots)++;
+ } else {
+ (*freed_slots)++;
+ }
+ heap_page_add_freeobj(objspace, sweep_page, dest);
+ gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(dest));
+ } else {
+ moved_slots++;
+ }
+ }
+ }
+ }
+ p++;
+ bitset >>= 1;
+ } while (bitset);
+ }
+ }
+
+ lock_page_body(objspace, GET_PAGE_BODY(heap->compact_cursor->start));
+
+ return finished_compacting;
+}
+
static inline int
gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page)
{
int i;
int empty_slots = 0, freed_slots = 0, final_slots = 0;
+ int was_compacting = 0;
RVALUE *p, *offset;
bits_t *bits, bitset;
gc_report(2, objspace, "page_sweep: start.\n");
+ if (heap->compact_cursor) {
+ if (sweep_page == heap->compact_cursor) {
+ /* The compaction cursor and sweep page met, so we need to quit compacting */
+ gc_report(5, objspace, "Quit compacting, mark and compact cursor met\n");
+ gc_compact_finish(objspace, heap);
+ } else {
+ /* We anticipate filling the page, so NULL out the freelist. */
+ asan_unpoison_memory_region(&sweep_page->freelist, sizeof(RVALUE*), false);
+ sweep_page->freelist = 0;
+ asan_poison_memory_region(&sweep_page->freelist, sizeof(RVALUE*));
+ was_compacting = 1;
+ }
+ }
+
sweep_page->flags.before_sweep = FALSE;
p = sweep_page->start;
@@ -4406,20 +4778,54 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
final_slots++;
}
else {
- (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
- heap_page_add_freeobj(objspace, sweep_page, vp);
- gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
- freed_slots++;
- asan_poison_object(vp);
+ if (heap->compact_cursor) {
+ /* We *want* to fill this slot */
+ MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(vp), vp);
+ } else {
+ (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
+ heap_page_add_freeobj(objspace, sweep_page, vp);
+ gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
+ freed_slots++;
+ }
+
}
break;
/* minor cases */
+ case T_MOVED:
+ if (objspace->flags.during_compacting) {
+ /* The sweep cursor shouldn't have made it to any
+ * T_MOVED slots while the compact flag is enabled.
+ * The sweep cursor and compact cursor move in
+ * opposite directions, and when they meet references will
+ * get updated and "during_compacting" should get disabled */
+ rb_bug("T_MOVED shouldn't be seen until compaction is finished\n");
+ }
+ gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
+ if (FL_TEST(vp, FL_FROM_FREELIST)) {
+ empty_slots++;
+ } else {
+ freed_slots++;
+ }
+ heap_page_add_freeobj(objspace, sweep_page, vp);
+ break;
case T_ZOMBIE:
/* already counted */
break;
case T_NONE:
- empty_slots++; /* already freed */
+ if (heap->compact_cursor) {
+ /* We *want* to fill this slot */
+ MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(vp), vp);
+ } else {
+ /* When we started sweeping this page, we were in
+ * compacting mode and nulled the free list for
+ * the page. But compaction finished, so we need to
+ * put any T_NONE slots back on the freelist. */
+ if (was_compacting) {
+ heap_page_add_freeobj(objspace, sweep_page, vp);
+ }
+ empty_slots++; /* already freed */
+ }
break;
}
}
@@ -4429,7 +4835,15 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
}
}
- gc_setup_mark_bits(sweep_page);
+ if (heap->compact_cursor) {
+ if (gc_fill_swept_page(objspace, heap, sweep_page, &freed_slots, &empty_slots)) {
+ gc_compact_finish(objspace, heap);
+ }
+ }
+
+ if (!heap->compact_cursor) {
+ gc_setup_mark_bits(sweep_page);
+ }
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
@@ -4642,6 +5056,85 @@ gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap)
}
static void
+invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
+{
+ int i;
+ int empty_slots = 0, freed_slots = 0;
+ bits_t *mark_bits, *pin_bits;
+ bits_t bitset;
+ RVALUE *p, *offset;
+
+ mark_bits = page->mark_bits;
+ pin_bits = page->pinned_bits;
+
+ p = page->start;
+ offset = p - NUM_IN_PAGE(p);
+
+ for (i=0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
+ /* Moved objects are pinned but never marked. We reuse the pin bits
+ * to indicate there is a moved object in this slot. */
+ bitset = pin_bits[i] & ~mark_bits[i];
+
+ if (bitset) {
+ p = offset + i * BITS_BITLENGTH;
+ do {
+ if (bitset & 1) {
+ VALUE forwarding_object = (VALUE)p;
+ VALUE object;
+
+ if (BUILTIN_TYPE(forwarding_object) == T_MOVED) {
+ GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object));
+ GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(forwarding_object), forwarding_object));
+
+ CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
+
+ object = rb_gc_location(forwarding_object);
+
+ if (FL_TEST(forwarding_object, FL_FROM_FREELIST)) {
+ empty_slots++; /* already freed */
+ } else {
+ freed_slots++;
+ }
+
+ gc_move(objspace, object, forwarding_object);
+ /* forwarding_object is now our actual object, and "object"
+ * is the free slot for the original page */
+ heap_page_add_freeobj(objspace, GET_HEAP_PAGE(object), object);
+
+ GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(forwarding_object), forwarding_object));
+ GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED);
+ GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
+ }
+ }
+ p++;
+ bitset >>= 1;
+ } while (bitset);
+ }
+ }
+
+ page->free_slots += (empty_slots + freed_slots);
+ objspace->profile.total_freed_objects += freed_slots;
+}
+
+static void
+gc_compact_start(rb_objspace_t *objspace, rb_heap_t *heap)
+{
+ heap->compact_cursor = list_tail(&heap->pages, struct heap_page, page_node);
+ heap->compact_cursor_index = 0;
+
+ if (gc_prof_enabled(objspace)) {
+ gc_profile_record *record = gc_prof_record(objspace);
+ record->moved_objects = objspace->rcompactor.total_moved;
+ }
+
+ memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
+ memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
+
+ /* Set up read barrier for pages containing MOVED objects */
+ install_handlers();
+}
+
+static void
gc_sweep(rb_objspace_t *objspace)
{
const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
@@ -4653,6 +5146,16 @@ gc_sweep(rb_objspace_t *objspace)
gc_prof_sweep_timer_start(objspace);
#endif
gc_sweep_start(objspace);
+ if (objspace->flags.during_compacting) {
+ struct heap_page *page = NULL;
+
+ list_for_each(&heap_eden->pages, page, page_node) {
+ page->flags.before_sweep = TRUE;
+ }
+
+ gc_compact_start(objspace, heap_eden);
+ }
+
gc_sweep_rest(objspace);
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
@@ -4662,6 +5165,10 @@ gc_sweep(rb_objspace_t *objspace)
struct heap_page *page = NULL;
gc_sweep_start(objspace);
+ if (ruby_enable_autocompact && is_full_marking(objspace)) {
+ gc_compact_start(objspace, heap_eden);
+ }
+
list_for_each(&heap_eden->pages, page, page_node) {
page->flags.before_sweep = TRUE;
}
@@ -5296,7 +5803,6 @@ gc_mark_maybe(rb_objspace_t *objspace, VALUE obj)
/* Garbage can live on the stack, so do not mark or pin */
switch (BUILTIN_TYPE(obj)) {
- case T_MOVED:
case T_ZOMBIE:
case T_NONE:
break;
@@ -6519,6 +7025,9 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark)
objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots);
#endif
objspace->flags.during_minor_gc = FALSE;
+ if (ruby_enable_autocompact) {
+ objspace->flags.during_compacting = TRUE;
+ }
objspace->profile.major_gc_count++;
objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
objspace->rgengc.old_objects = 0;
@@ -7076,6 +7585,10 @@ gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
gc_remember_unprotected(objspace, b);
}
}
+
+ if (UNLIKELY(objspace->flags.during_compacting)) {
+ MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
+ }
}
}
#else
@@ -7309,6 +7822,7 @@ rb_gc_force_recycle(VALUE obj)
}
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
+ CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
#if GC_ENABLE_INCREMENTAL_MARK
if (is_incremental_marking(objspace)) {
@@ -7539,6 +8053,7 @@ gc_start(rb_objspace_t *objspace, int reason)
/* reason may be clobbered, later, so keep set immediate_sweep here */
objspace->flags.immediate_sweep = !!((unsigned)reason & GPR_FLAG_IMMEDIATE_SWEEP);
+ objspace->flags.during_compacting = !!((unsigned)reason & GPR_FLAG_COMPACT);
if (!heap_allocated_pages) return FALSE; /* heap is not ready */
if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
@@ -7809,7 +8324,7 @@ garbage_collect_with_gvl(rb_objspace_t *objspace, int reason)
}
static VALUE
-gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep)
+gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep, VALUE compact)
{
rb_objspace_t *objspace = &rb_objspace;
int reason = GPR_FLAG_FULL_MARK |
@@ -7817,9 +8332,14 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE
GPR_FLAG_IMMEDIATE_SWEEP |
GPR_FLAG_METHOD;
- if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
- if (!RTEST(immediate_mark)) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
- if (!RTEST(immediate_sweep)) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
+ /* For now, compact implies full mark / sweep, so ignore other flags */
+ if (RTEST(compact)) {
+ reason |= GPR_FLAG_COMPACT;
+ } else {
+ if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
+ if (!RTEST(immediate_mark)) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
+ if (!RTEST(immediate_sweep)) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
+ }
garbage_collect(objspace, reason);
gc_finalize_deferred(objspace);
@@ -7830,9 +8350,7 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE
static int
gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
{
- if (SPECIAL_CONST_P(obj)) {
- return FALSE;
- }
+ GC_ASSERT(!SPECIAL_CONST_P(obj));
switch (BUILTIN_TYPE(obj)) {
case T_NONE:
@@ -7873,7 +8391,10 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
return FALSE;
}
}
- return RVALUE_MARKED(obj) && !RVALUE_PINNED(obj);
+ GC_ASSERT(RVALUE_MARKED(obj));
+ GC_ASSERT(!RVALUE_PINNED(obj));
+
+ return TRUE;
default:
rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
@@ -7884,7 +8405,7 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
}
static VALUE
-gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct RMoved * moved_list)
+gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free)
{
int marked;
int wb_unprotected;
@@ -7896,7 +8417,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct RMoved * moved_l
gc_report(4, objspace, "Moving object: %p -> %p\n", (void*)scan, (void *)free);
GC_ASSERT(BUILTIN_TYPE(scan) != T_NONE);
- GC_ASSERT(BUILTIN_TYPE(free) == T_NONE);
+ GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(free), free));
/* Save off bits for current object. */
marked = rb_objspace_marked_object_p((VALUE)src);
@@ -7904,8 +8425,6 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct RMoved * moved_l
uncollectible = RVALUE_UNCOLLECTIBLE((VALUE)src);
marking = RVALUE_MARKING((VALUE)src);
- objspace->total_allocated_objects++;
-
/* Clear bits for eventual T_MOVED */
CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)src), (VALUE)src);
CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS((VALUE)src), (VALUE)src);
@@ -7913,7 +8432,11 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct RMoved * moved_l
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS((VALUE)src), (VALUE)src);
if (FL_TEST((VALUE)src, FL_EXIVAR)) {
+ /* Same deal as below. Generic ivars are held in st tables.
+ * Resizing the table could cause a GC to happen and we can't allow it */
+ VALUE already_disabled = rb_gc_disable_no_rest();
rb_mv_generic_ivar((VALUE)src, (VALUE)dest);
+ if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
}
st_data_t srcid = (st_data_t)src, id;
@@ -7922,8 +8445,13 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct RMoved * moved_l
* the object to object id mapping. */
if (st_lookup(objspace->obj_to_id_tbl, srcid, &id)) {
gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest);
+ /* inserting in the st table can cause the GC to run. We need to
+ * prevent re-entry in to the GC since `gc_move` is running in the GC,
+ * so temporarily disable the GC around the st table mutation */
+ VALUE already_disabled = rb_gc_disable_no_rest();
st_delete(objspace->obj_to_id_tbl, &srcid, 0);
st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id);
+ if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
}
/* Move the object */
@@ -7961,102 +8489,13 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, struct RMoved * moved_l
/* Assign forwarding address */
src->as.moved.flags = T_MOVED;
+ src->as.moved.dummy = Qundef;
src->as.moved.destination = (VALUE)dest;
- list_add(&moved_list->as.head, &src->as.moved.as.node);
GC_ASSERT(BUILTIN_TYPE((VALUE)dest) != T_NONE);
return (VALUE)src;
}
-struct heap_cursor {
- RVALUE *slot;
- size_t index;
- struct heap_page *page;
- rb_objspace_t * objspace;
-};
-
-static void
-advance_cursor(struct heap_cursor *free, struct heap_page **page_list)
-{
- if (free->slot == free->page->start + free->page->total_slots - 1) {
- free->index++;
- free->page = page_list[free->index];
- free->slot = free->page->start;
- }
- else {
- free->slot++;
- }
-}
-
-static void
-retreat_cursor(struct heap_cursor *scan, struct heap_page **page_list)
-{
- if (scan->slot == scan->page->start) {
- scan->index--;
- scan->page = page_list[scan->index];
- scan->slot = scan->page->start + scan->page->total_slots - 1;
- }
- else {
- scan->slot--;
- }
-}
-
-static int
-not_met(struct heap_cursor *free, struct heap_cursor *scan)
-{
- if (free->index < scan->index)
- return 1;
-
- if (free->index > scan->index)
- return 0;
-
- return free->slot < scan->slot;
-}
-
-static void
-init_cursors(rb_objspace_t *objspace, struct heap_cursor *free, struct heap_cursor *scan, struct heap_page **page_list)
-{
- struct heap_page *page;
- size_t total_pages = heap_eden->total_pages;
- page = page_list[0];
-
- free->index = 0;
- free->page = page;
- free->slot = page->start;
- free->objspace = objspace;
-
- page = page_list[total_pages - 1];
- scan->index = total_pages - 1;
- scan->page = page;
- scan->slot = page->start + page->total_slots - 1;
- scan->objspace = objspace;
-}
-
-static int
-count_pinned(struct heap_page *page)
-{
- int pinned = 0;
- int i;
-
- for (i = 0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
- pinned += popcount_bits(page->pinned_bits[i]);
- }
-
- return pinned;
-}
-
-static int
-compare_pinned(const void *left, const void *right, void *dummy)
-{
- struct heap_page *left_page;
- struct heap_page *right_page;
-
- left_page = *(struct heap_page * const *)left;
- right_page = *(struct heap_page * const *)right;
-
- return right_page->pinned_slots - left_page->pinned_slots;
-}
-
static int
compare_free_slots(const void *left, const void *right, void *dummy)
{
@@ -8066,110 +8505,56 @@ compare_free_slots(const void *left, const void *right, void *dummy)
left_page = *(struct heap_page * const *)left;
right_page = *(struct heap_page * const *)right;
- return right_page->free_slots - left_page->free_slots;
+ return left_page->free_slots - right_page->free_slots;
}
-typedef int page_compare_func_t(const void *, const void *, void *);
-
-static struct heap_page **
-allocate_page_list(rb_objspace_t *objspace, page_compare_func_t *comparator)
+static VALUE
+gc_sort_heap_by_empty_slots(rb_execution_context_t *ec, VALUE self)
{
+ rb_objspace_t *objspace = &rb_objspace;
+
size_t total_pages = heap_eden->total_pages;
size_t size = size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
struct heap_page *page = 0, **page_list = malloc(size);
- int i = 0;
+ size_t i = 0;
+
+ gc_rest(objspace);
list_for_each(&heap_eden->pages, page, page_node) {
page_list[i++] = page;
- page->pinned_slots = count_pinned(page);
GC_ASSERT(page != NULL);
}
GC_ASSERT(total_pages > 0);
GC_ASSERT((size_t)i == total_pages);
- ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), comparator, NULL);
+ /* Sort the heap so "filled pages" are first. `heap_add_page` adds to the
+ * head of the list, so empty pages will end up at the start of the heap */
+ ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_free_slots, NULL);
- return page_list;
-}
-
-static void
-gc_compact_heap(rb_objspace_t *objspace, page_compare_func_t *comparator, struct RMoved * moved_list)
-{
- struct heap_cursor free_cursor;
- struct heap_cursor scan_cursor;
- struct heap_page **page_list;
-
- memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
- memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
-
- page_list = allocate_page_list(objspace, comparator);
-
- init_cursors(objspace, &free_cursor, &scan_cursor, page_list);
-
- /* Two finger algorithm */
- while (not_met(&free_cursor, &scan_cursor)) {
- /* Free cursor movement */
-
- /* Unpoison free_cursor slot */
- void *free_slot_poison = asan_poisoned_object_p((VALUE)free_cursor.slot);
- asan_unpoison_object((VALUE)free_cursor.slot, false);
-
- while (BUILTIN_TYPE((VALUE)free_cursor.slot) != T_NONE && not_met(&free_cursor, &scan_cursor)) {
- /* Re-poison slot if it's not the one we want */
- if (free_slot_poison) {
- GC_ASSERT(BUILTIN_TYPE((VALUE)free_cursor.slot) == T_NONE);
- asan_poison_object((VALUE)free_cursor.slot);
- }
-
- advance_cursor(&free_cursor, page_list);
-
- /* Unpoison free_cursor slot */
- free_slot_poison = asan_poisoned_object_p((VALUE)free_cursor.slot);
- asan_unpoison_object((VALUE)free_cursor.slot, false);
- }
-
- /* Unpoison scan_cursor slot */
- void *scan_slot_poison = asan_poisoned_object_p((VALUE)scan_cursor.slot);
- asan_unpoison_object((VALUE)scan_cursor.slot, false);
-
- /* Scan cursor movement */
- objspace->rcompactor.considered_count_table[BUILTIN_TYPE((VALUE)scan_cursor.slot)]++;
-
- while (!gc_is_moveable_obj(objspace, (VALUE)scan_cursor.slot) && not_met(&free_cursor, &scan_cursor)) {
-
- /* Re-poison slot if it's not the one we want */
- if (scan_slot_poison) {
- GC_ASSERT(BUILTIN_TYPE((VALUE)scan_cursor.slot) == T_NONE);
- asan_poison_object((VALUE)scan_cursor.slot);
- }
-
- retreat_cursor(&scan_cursor, page_list);
-
- /* Unpoison scan_cursor slot */
- scan_slot_poison = asan_poisoned_object_p((VALUE)scan_cursor.slot);
- asan_unpoison_object((VALUE)scan_cursor.slot, false);
+ /* Reset the eden heap */
+ list_head_init(&objspace->eden_heap.pages);
+ heap_eden->free_pages = NULL;
- objspace->rcompactor.considered_count_table[BUILTIN_TYPE((VALUE)scan_cursor.slot)]++;
+ for (i = 0; i < total_pages; i++) {
+ list_add(&heap_eden->pages, &page_list[i]->page_node);
+ if (page_list[i]->free_slots != 0) {
+ heap_add_freepage(heap_eden, page_list[i]);
}
+ }
- if (not_met(&free_cursor, &scan_cursor)) {
- objspace->rcompactor.moved_count_table[BUILTIN_TYPE((VALUE)scan_cursor.slot)]++;
+ free(page_list);
- GC_ASSERT(BUILTIN_TYPE((VALUE)free_cursor.slot) == T_NONE);
- GC_ASSERT(BUILTIN_TYPE((VALUE)scan_cursor.slot) != T_NONE);
- GC_ASSERT(BUILTIN_TYPE((VALUE)scan_cursor.slot) != T_MOVED);
+ return Qnil;
+}
- gc_move(objspace, (VALUE)scan_cursor.slot, (VALUE)free_cursor.slot, moved_list);
+static VALUE
+gc_double_heap_size(rb_execution_context_t *ec, VALUE self)
+{
+ rb_objspace_t *objspace = &rb_objspace;
- GC_ASSERT(BUILTIN_TYPE((VALUE)free_cursor.slot) != T_MOVED);
- GC_ASSERT(BUILTIN_TYPE((VALUE)free_cursor.slot) != T_NONE);
- GC_ASSERT(BUILTIN_TYPE((VALUE)scan_cursor.slot) == T_MOVED);
+ heap_add_pages(objspace, heap_eden, heap_allocated_pages);
- advance_cursor(&free_cursor, page_list);
- retreat_cursor(&scan_cursor, page_list);
- }
- }
- free(page_list);
+ return Qnil;
}
static void
@@ -8387,8 +8772,19 @@ gc_ref_update_imemo(rb_objspace_t *objspace, VALUE obj)
case imemo_callcache:
{
const struct rb_callcache *cc = (const struct rb_callcache *)obj;
- UPDATE_IF_MOVED(objspace, cc->klass);
- TYPED_UPDATE_IF_MOVED(objspace, struct rb_callable_method_entry_struct *, cc->cme_);
+ if (cc->klass) {
+ UPDATE_IF_MOVED(objspace, cc->klass);
+ if (!is_live_object(objspace, cc->klass)) {
+ *((VALUE *)(&cc->klass)) = (VALUE)0;
+ }
+ }
+
+ if (cc->cme_) {
+ TYPED_UPDATE_IF_MOVED(objspace, struct rb_callable_method_entry_struct *, cc->cme_);
+ if (!is_live_object(objspace, (VALUE)cc->cme_)) {
+ *((struct rb_callable_method_entry_struct **)(&cc->cme_)) = (struct rb_callable_method_entry_struct *)0;
+ }
+ }
}
break;
case imemo_parser_strterm:
@@ -8713,17 +9109,10 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
}
static int
-gc_ref_update(void *vstart, void *vend, size_t stride, void * data)
+gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t * objspace, struct heap_page *page)
{
- rb_objspace_t * objspace;
- struct heap_page *page;
- short free_slots = 0;
-
VALUE v = (VALUE)vstart;
- objspace = (rb_objspace_t *)data;
- page = GET_HEAP_PAGE(v);
asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
- page->freelist = NULL;
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
page->flags.has_uncollectible_shady_objects = FALSE;
page->flags.has_remembered_objects = FALSE;
@@ -8735,11 +9124,7 @@ gc_ref_update(void *vstart, void *vend, size_t stride, void * data)
switch (BUILTIN_TYPE(v)) {
case T_NONE:
- heap_page_add_freeobj(objspace, page, v);
- free_slots++;
- break;
case T_MOVED:
- break;
case T_ZOMBIE:
break;
default:
@@ -8749,7 +9134,13 @@ gc_ref_update(void *vstart, void *vend, size_t stride, void * data)
if (RVALUE_PAGE_MARKING(page, v)) {
page->flags.has_remembered_objects = TRUE;
}
- gc_update_object_references(objspace, v);
+ if (page->flags.before_sweep) {
+ if (RVALUE_MARKED(v)) {
+ gc_update_object_references(objspace, v);
+ }
+ } else {
+ gc_update_object_references(objspace, v);
+ }
}
if (poisoned) {
@@ -8757,7 +9148,6 @@ gc_ref_update(void *vstart, void *vend, size_t stride, void * data)
}
}
- page->free_slots = free_slots;
return 0;
}
@@ -8765,12 +9155,23 @@ extern rb_symbols_t ruby_global_symbols;
#define global_symbols ruby_global_symbols
static void
-gc_update_references(rb_objspace_t * objspace)
+gc_update_references(rb_objspace_t * objspace, rb_heap_t *heap)
{
rb_execution_context_t *ec = GET_EC();
rb_vm_t *vm = rb_ec_vm_ptr(ec);
+ short should_set_mark_bits = 1;
- objspace_each_objects_without_setup(objspace, gc_ref_update, objspace);
+ struct heap_page *page = NULL;
+
+ list_for_each(&heap->pages, page, page_node) {
+ gc_ref_update(page->start, page->start + page->total_slots, sizeof(RVALUE), objspace, page);
+ if (page == heap->sweeping_page) {
+ should_set_mark_bits = 0;
+ }
+ if (should_set_mark_bits) {
+ gc_setup_mark_bits(page);
+ }
+ }
rb_vm_update_references(vm);
rb_transient_heap_update_references();
rb_gc_update_global_tbl();
@@ -8785,9 +9186,10 @@ gc_update_references(rb_objspace_t * objspace)
static VALUE type_sym(size_t type);
static VALUE
-gc_compact_stats(rb_objspace_t *objspace)
+gc_compact_stats(rb_execution_context_t *ec, VALUE self)
{
size_t i;
+ rb_objspace_t *objspace = &rb_objspace;
VALUE h = rb_hash_new();
VALUE considered = rb_hash_new();
VALUE moved = rb_hash_new();
@@ -8808,34 +9210,6 @@ gc_compact_stats(rb_objspace_t *objspace)
return h;
}
-static void gc_compact_after_gc(rb_objspace_t *objspace, int use_toward_empty, int use_double_pages, int use_verifier);
-
-static void
-gc_compact(rb_objspace_t *objspace, int use_toward_empty, int use_double_pages, int use_verifier)
-{
-
- objspace->flags.during_compacting = TRUE;
- {
- mjit_gc_start_hook();
- /* pin objects referenced by maybe pointers */
- garbage_collect(objspace, GPR_DEFAULT_REASON);
- /* compact */
- gc_compact_after_gc(objspace, use_toward_empty, use_double_pages, use_verifier);
- mjit_gc_exit_hook();
- }
- objspace->flags.during_compacting = FALSE;
-}
-
-static VALUE
-rb_gc_compact(rb_execution_context_t *ec, VALUE self)
-{
- rb_objspace_t *objspace = &rb_objspace;
- if (dont_gc_val()) return Qnil;
-
- gc_compact(objspace, FALSE, FALSE, FALSE);
- return gc_compact_stats(objspace);
-}
-
static void
root_obj_check_moved_i(const char *category, VALUE obj, void *data)
{
@@ -8870,7 +9244,9 @@ heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
case T_ZOMBIE:
break;
default:
- rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v);
+ if (!rb_objspace_garbage_object_p(v)) {
+ rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v);
+ }
}
if (poisoned) {
@@ -8884,119 +9260,14 @@ heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
}
static VALUE
-gc_check_references_for_moved(rb_objspace_t *objspace)
+gc_check_references_for_moved(rb_execution_context_t *ec, VALUE self)
{
+ rb_objspace_t *objspace = &rb_objspace;
objspace_reachable_objects_from_root(objspace, root_obj_check_moved_i, NULL);
objspace_each_objects(objspace, heap_check_moved_i, NULL);
return Qnil;
}
-static void
-gc_unlink_moved_list(rb_objspace_t *objspace, struct RMoved * moved_list_head)
-{
- /* For each moved slot */
- struct RMoved * obj = NULL, *next;
- list_for_each_safe(&RMOVED(moved_list_head)->as.head, obj, next, as.node) {
- if (obj != moved_list_head) {
- struct heap_page *page;
-
- page = GET_HEAP_PAGE((VALUE)obj);
-
- /* clear the memory for that moved slot */
- memset(obj, 0, sizeof(VALUE));
- page->free_slots++;
- heap_page_add_freeobj(objspace, page, (VALUE)obj);
-
- if (page->free_slots == page->total_slots && heap_pages_freeable_pages > 0) {
- heap_pages_freeable_pages--;
- heap_unlink_page(objspace, heap_eden, page);
- heap_add_page(objspace, heap_tomb, page);
- }
- objspace->profile.total_freed_objects++;
- }
- }
-}
-
-static void
-gc_compact_after_gc(rb_objspace_t *objspace, int use_toward_empty, int use_double_pages, int use_verifier)
-{
- if (0) fprintf(stderr, "gc_compact_after_gc: %d,%d,%d\n", use_toward_empty, use_double_pages, use_verifier);
-
- mjit_gc_start_hook(); // prevent MJIT from running while moving pointers related to ISeq
-
- objspace->profile.compact_count++;
-
- if (use_verifier) {
- gc_verify_internal_consistency(objspace);
- }
-
- if (use_double_pages) {
- /* Double heap size */
- heap_add_pages(objspace, heap_eden, heap_allocated_pages);
- }
-
- struct RMoved moved_list_head;
- list_head_init(&moved_list_head.as.head);
-
- VALUE disabled = rb_objspace_gc_disable(objspace);
-
- if (use_toward_empty) {
- gc_compact_heap(objspace, compare_free_slots, &moved_list_head);
- }
- else {
- gc_compact_heap(objspace, compare_pinned, &moved_list_head);
- }
- heap_eden->freelist = NULL;
-
- gc_update_references(objspace);
- if (!RTEST(disabled)) rb_objspace_gc_enable(objspace);
-
- if (use_verifier) {
- gc_check_references_for_moved(objspace);
- }
-
- rb_clear_constant_cache();
- heap_eden->free_pages = NULL;
- heap_eden->using_page = NULL;
-
- gc_unlink_moved_list(objspace, &moved_list_head);
-
- /* Add any eden pages with free slots back to the free pages list */
- struct heap_page *page = NULL;
- list_for_each(&heap_eden->pages, page, page_node) {
- if (page->free_slots > 0) {
- heap_add_freepage(heap_eden, page);
- } else {
- page->free_next = NULL;
- }
- }
-
- /* Set up "using_page" if we have any pages with free slots */
- if (heap_eden->free_pages) {
- heap_eden->using_page = heap_eden->free_pages;
- heap_eden->free_pages = heap_eden->free_pages->free_next;
- }
-
- if (use_verifier) {
- gc_verify_internal_consistency(objspace);
- }
-
- mjit_gc_exit_hook(); // unlock MJIT here, because `rb_gc()` calls `mjit_gc_start_hook()` again.
-}
-
-static VALUE
-gc_verify_compaction_references(rb_execution_context_t *ec, VALUE mod, VALUE toward, VALUE double_heap)
-{
- rb_objspace_t *objspace = &rb_objspace;
- const ID id_empty = rb_intern("empty");
- const int use_toward_empty = NIL_P(toward) ? FALSE :
- (Check_Type(toward, T_SYMBOL), toward == ID2SYM(id_empty));
- const int use_double_pages = RTEST(double_heap);
-
- gc_compact(objspace, use_toward_empty, use_double_pages, TRUE);
- return gc_compact_stats(objspace);
-}
-
VALUE
rb_gc_start(void)
{
@@ -9184,6 +9455,8 @@ enum gc_stat_sym {
gc_stat_sym_minor_gc_count,
gc_stat_sym_major_gc_count,
gc_stat_sym_compact_count,
+ gc_stat_sym_read_barrier_faults,
+ gc_stat_sym_total_moved_objects,
gc_stat_sym_remembered_wb_unprotected_objects,
gc_stat_sym_remembered_wb_unprotected_objects_limit,
gc_stat_sym_old_objects,
@@ -9257,6 +9530,8 @@ setup_gc_stat_symbols(void)
S(minor_gc_count);
S(major_gc_count);
S(compact_count);
+ S(read_barrier_faults);
+ S(total_moved_objects);
S(remembered_wb_unprotected_objects);
S(remembered_wb_unprotected_objects_limit);
S(old_objects);
@@ -9424,6 +9699,8 @@ gc_stat_internal(VALUE hash_or_sym)
SET(minor_gc_count, objspace->profile.minor_gc_count);
SET(major_gc_count, objspace->profile.major_gc_count);
SET(compact_count, objspace->profile.compact_count);
+ SET(read_barrier_faults, objspace->profile.read_barrier_faults);
+ SET(total_moved_objects, objspace->rcompactor.total_moved);
SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
SET(old_objects, objspace->rgengc.old_objects);
@@ -9579,6 +9856,19 @@ gc_disable(rb_execution_context_t *ec, VALUE _)
return rb_gc_disable();
}
+static VALUE
+gc_set_auto_compact(rb_execution_context_t *ec, VALUE _, VALUE v)
+{
+ ruby_enable_autocompact = RTEST(v);
+ return v;
+}
+
+static VALUE
+gc_get_auto_compact(rb_execution_context_t *ec, VALUE _)
+{
+ return ruby_enable_autocompact ? Qtrue : Qfalse;
+}
+
static int
get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
{
@@ -11450,6 +11740,7 @@ gc_profile_record_get(VALUE _)
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
+ rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
#if GC_PROFILE_MORE_DETAIL
rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
diff --git a/gc.rb b/gc.rb
index 8bba0dfbc9..c763fc1359 100644
--- a/gc.rb
+++ b/gc.rb
@@ -31,11 +31,32 @@ module GC
# are not guaranteed to be future-compatible, and may be ignored if the
# underlying implementation does not support them.
def self.start full_mark: true, immediate_mark: true, immediate_sweep: true
- Primitive.gc_start_internal full_mark, immediate_mark, immediate_sweep
+ Primitive.gc_start_internal full_mark, immediate_mark, immediate_sweep, false
end
def garbage_collect full_mark: true, immediate_mark: true, immediate_sweep: true
- Primitive.gc_start_internal full_mark, immediate_mark, immediate_sweep
+ Primitive.gc_start_internal full_mark, immediate_mark, immediate_sweep, false
+ end
+
+ # call-seq:
+ # GC.auto_compact -> true or false
+ #
+ # Returns whether or not automatic compaction has been enabled.
+ #
+ def self.auto_compact
+ Primitive.gc_get_auto_compact
+ end
+
+ # call-seq:
+ # GC.auto_compact = flag
+ #
+ # Updates automatic compaction mode.
+ #
+ # When enabled, the compactor will execute on every major collection.
+ #
+ # Enabling compaction will degrade performance on major collections.
+ def self.auto_compact=(flag)
+ Primitive.gc_set_auto_compact(flag)
end
# call-seq:
@@ -163,8 +184,23 @@ module GC
Primitive.gc_latest_gc_info hash_or_key
end
+ # call-seq:
+ # GC.latest_compact_info -> {:considered=>{:T_CLASS=>11}, :moved=>{:T_CLASS=>11}}
+ #
+ # Returns information about object moved in the most recent GC compaction.
+ #
+ # The returned hash has two keys :considered and :moved. The hash for
+ # :considered lists the number of objects that were considered for movement
+ # by the compactor, and the :moved hash lists the number of objects that
+ # were actually moved. Some objects can't be moved (maybe they were pinned)
+ # so these numbers can be used to calculate compaction efficiency.
+ def self.latest_compact_info
+ Primitive.gc_compact_stats
+ end
+
def self.compact
- Primitive.rb_gc_compact
+ Primitive.gc_start_internal true, true, true, true
+ Primitive.gc_compact_stats
end
# call-seq:
@@ -182,13 +218,23 @@ module GC
# object, that object should be pushed on the mark stack, and will
# make a SEGV.
def self.verify_compaction_references(toward: nil, double_heap: false)
- Primitive.gc_verify_compaction_references(toward, double_heap)
+ if double_heap
+ Primitive.gc_double_heap_size
+ end
+
+ if toward == :empty
+ Primitive.gc_sort_heap_by_empty_slots
+ end
+
+ Primitive.gc_start_internal true, true, true, true
+ Primitive.gc_check_references_for_moved
+ Primitive.gc_compact_stats
end
end
module ObjectSpace
def garbage_collect full_mark: true, immediate_mark: true, immediate_sweep: true
- Primitive.gc_start_internal full_mark, immediate_mark, immediate_sweep
+ Primitive.gc_start_internal full_mark, immediate_mark, immediate_sweep, false
end
module_function :garbage_collect
diff --git a/internal/vm.h b/internal/vm.h
index 1ae2ba064a..246a42a793 100644
--- a/internal/vm.h
+++ b/internal/vm.h
@@ -47,6 +47,7 @@ rb_serial_t rb_next_class_serial(void);
/* vm.c */
VALUE rb_obj_is_thread(VALUE obj);
void rb_vm_mark(void *ptr);
+void rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx);
PUREFUNC(VALUE rb_vm_top_self(void));
void rb_vm_inc_const_missing_count(void);
const void **rb_vm_get_insns_address_table(void);
diff --git a/proc.c b/proc.c
index 5c78fc10ed..1ac07b10a4 100644
--- a/proc.c
+++ b/proc.c
@@ -67,7 +67,7 @@ block_mark(const struct rb_block *block)
RUBY_MARK_MOVABLE_UNLESS_NULL(captured->self);
RUBY_MARK_MOVABLE_UNLESS_NULL((VALUE)captured->code.val);
if (captured->ep && captured->ep[VM_ENV_DATA_INDEX_ENV] != Qundef /* cfunc_proc_t */) {
- RUBY_MARK_MOVABLE_UNLESS_NULL(VM_ENV_ENVVAL(captured->ep));
+ rb_gc_mark(VM_ENV_ENVVAL(captured->ep));
}
}
break;
@@ -90,9 +90,6 @@ block_compact(struct rb_block *block)
struct rb_captured_block *captured = &block->as.captured;
captured->self = rb_gc_location(captured->self);
captured->code.val = rb_gc_location(captured->code.val);
- if (captured->ep && captured->ep[VM_ENV_DATA_INDEX_ENV] != Qundef /* cfunc_proc_t */) {
- UPDATE_REFERENCE(captured->ep[VM_ENV_DATA_INDEX_ENV]);
- }
}
break;
case block_type_symbol:
diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb
index 75d9b01f2c..0ceb54dba7 100644
--- a/test/ruby/test_gc_compact.rb
+++ b/test/ruby/test_gc_compact.rb
@@ -3,6 +3,71 @@ require 'test/unit'
require 'fiddle'
class TestGCCompact < Test::Unit::TestCase
+ def test_enable_autocompact
+ before = GC.auto_compact
+ GC.auto_compact = true
+ assert GC.auto_compact
+ ensure
+ GC.auto_compact = before
+ end
+
+ def test_disable_autocompact
+ before = GC.auto_compact
+ GC.auto_compact = false
+ refute GC.auto_compact
+ ensure
+ GC.auto_compact = before
+ end
+
+ def test_major_compacts
+ before = GC.auto_compact
+ GC.auto_compact = true
+ compact = GC.stat :compact_count
+ GC.start
+ assert_operator GC.stat(:compact_count), :>, compact
+ ensure
+ GC.auto_compact = before
+ end
+
+ def test_implicit_compaction_does_something
+ before = GC.auto_compact
+ list = []
+ list2 = []
+
+ # Try to make some fragmentation
+ 500.times {
+ list << Object.new
+ Object.new
+ Object.new
+ }
+ count = GC.stat :compact_count
+ GC.auto_compact = true
+ loop do
+ break if count < GC.stat(:compact_count)
+ list2 << Object.new
+ end
+ compact_stats = GC.latest_compact_info
+ refute_predicate compact_stats[:considered], :empty?
+ refute_predicate compact_stats[:moved], :empty?
+ ensure
+ GC.auto_compact = before
+ end
+
+ def test_gc_compact_stats
+ list = []
+ list2 = []
+
+ # Try to make some fragmentation
+ 500.times {
+ list << Object.new
+ Object.new
+ Object.new
+ }
+ compact_stats = GC.compact
+ refute_predicate compact_stats[:considered], :empty?
+ refute_predicate compact_stats[:moved], :empty?
+ end
+
def memory_location(obj)
(Fiddle.dlwrap(obj) >> 1)
end
diff --git a/transient_heap.c b/transient_heap.c
index 444f4fdcc4..51886d8dc1 100644
--- a/transient_heap.c
+++ b/transient_heap.c
@@ -864,26 +864,17 @@ blocks_clear_marked_index(struct transient_heap_block* block)
static void
transient_heap_block_update_refs(struct transient_heap* theap, struct transient_heap_block* block)
{
- int i=0, n=0;
+ int marked_index = block->info.last_marked_index;
- while (i<block->info.index) {
- void *ptr = &block->buff[i];
- struct transient_alloc_header *header = ptr;
+ while (marked_index >= 0) {
+ struct transient_alloc_header *header = alloc_header(block, marked_index);
asan_unpoison_memory_region(header, sizeof *header, false);
- void *poisoned = __asan_region_is_poisoned((void *)header->obj, SIZEOF_VALUE);
- asan_unpoison_object(header->obj, false);
-
header->obj = rb_gc_location(header->obj);
- if (poisoned) {
- asan_poison_object(header->obj);
- }
-
- i += header->size;
+ marked_index = header->next_marked_index;
asan_poison_memory_region(header, sizeof *header);
- n++;
}
}
diff --git a/vm.c b/vm.c
index ccb04c75f6..83911f8e14 100644
--- a/vm.c
+++ b/vm.c
@@ -2495,6 +2495,36 @@ rb_vm_update_references(void *ptr)
}
void
+rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx)
+{
+ if (ptr) {
+ rb_vm_t *vm = ptr;
+ rb_ractor_t *r = 0;
+ list_for_each(&vm->ractor.set, r, vmlr_node) {
+ VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) ||
+ rb_ractor_status_p(r, ractor_running));
+ if (r->threads.cnt > 0) {
+ rb_thread_t *th = 0;
+ list_for_each(&r->threads.set, th, lt_node) {
+ VM_ASSERT(th != NULL);
+ rb_execution_context_t * ec = th->ec;
+ if (ec->vm_stack) {
+ VALUE *p = ec->vm_stack;
+ VALUE *sp = ec->cfp->sp;
+ while (p <= sp) {
+ if (!rb_special_const_p(*p)) {
+ cb(*p, ctx);
+ }
+ p++;
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+void
rb_vm_mark(void *ptr)
{
RUBY_MARK_ENTER("vm");