summaryrefslogtreecommitdiff
path: root/transient_heap.c
diff options
context:
space:
mode:
authorko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-10-30 21:53:56 (GMT)
committerko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-10-30 21:53:56 (GMT)
commit312b105d0e263a36c129a14f2681fe16905c0244 (patch)
tree00bf11825e6f8a8cb12ba1f2c02c722f35090e0c /transient_heap.c
parent69b8ffcd5b8b28c56212368a3fe66f076b90e4c4 (diff)
introduce TransientHeap. [Bug #14858]
* transient_heap.c, transient_heap.h: implement TransientHeap (theap). theap is designed for Ruby's object system. theap is like Eden heap on generational GC terminology. theap allocation is very fast because it only needs to bump up pointer and deallocation is also fast because we don't do anything. However we need to evacuate (Copy GC terminology) if theap memory is long-lived. Evacuation logic is needed for each type. See [Bug #14858] for details. * array.c: Now, theap for T_ARRAY is supported. ary_heap_alloc() tries to allocate memory area from theap. If this trial sccesses, this array has theap ptr and RARRAY_TRANSIENT_FLAG is turned on. We don't need to free theap ptr. * ruby.h: RARRAY_CONST_PTR() returns malloc'ed memory area. It menas that if ary is allocated at theap, force evacuation to malloc'ed memory. It makes programs slow, but very compatible with current code because theap memory can be evacuated (theap memory will be recycled). If you want to get transient heap ptr, use RARRAY_CONST_PTR_TRANSIENT() instead of RARRAY_CONST_PTR(). If you can't understand when evacuation will occur, use RARRAY_CONST_PTR(). (re-commit of r65444) git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65449 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'transient_heap.c')
-rw-r--r--transient_heap.c834
1 files changed, 834 insertions, 0 deletions
diff --git a/transient_heap.c b/transient_heap.c
new file mode 100644
index 0000000..e9a2460
--- /dev/null
+++ b/transient_heap.c
@@ -0,0 +1,834 @@
+/**********************************************************************
+
+ transient_heap.c - implement transient_heap.
+
+ Copyright (C) 2018 Koichi Sasada
+
+**********************************************************************/
+
+#include "ruby/ruby.h"
+#include "ruby/debug.h"
+#include "vm_debug.h"
+#include "gc.h"
+#include "internal.h"
+#include "ruby_assert.h"
+#include "transient_heap.h"
+#include "debug_counter.h"
+
+/*
+ * 1: enable assertions
+ * 2: enable verify all transient heaps
+ */
+#ifndef TRANSIENT_HEAP_CHECK_MODE
+#define TRANSIENT_HEAP_CHECK_MODE 0
+#endif
+#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
+
+/*
+ * 1: show events
+ * 2: show dump at events
+ * 3: show all operations
+ */
+#define TRANSIENT_HEAP_DEBUG 0
+
+/* For Debug: Provide blocks infinitely.
+ * This mode generates blocks unlimitedly
+ * and prohibit access free'ed blocks to check invalid access.
+ */
+#define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
+
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+#include <sys/mman.h>
+#include <errno.h>
+#endif
+
+/* For Debug: Prohibit promoting to malloc space.
+ */
+#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
+
+/* size configuration */
+#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
+
+ /* K M */
+#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */
+#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */
+#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */
+#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
+
+#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
+#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
+
+#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
+#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
+
+enum transient_heap_status {
+ transient_heap_none,
+ transient_heap_marking,
+ transient_heap_escaping
+};
+
+struct transient_heap_block {
+ struct transient_heap_block_header {
+ int16_t size; /* sizeof(block) = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header) */
+ int16_t index;
+ int16_t last_marked_index;
+ int16_t objects;
+ struct transient_heap_block *next_block;
+ } info;
+ char buff[TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header)];
+};
+
+struct transient_heap {
+ struct transient_heap_block *using_blocks;
+ struct transient_heap_block *marked_blocks;
+ struct transient_heap_block *free_blocks;
+ int total_objects;
+ int total_marked_objects;
+ int total_blocks;
+ enum transient_heap_status status;
+
+ VALUE *promoted_objects;
+ int promoted_objects_size;
+ int promoted_objects_index;
+
+ struct transient_heap_block *arena;
+ int arena_index; /* increment only */
+};
+
+struct transient_alloc_header {
+ uint16_t magic;
+ uint16_t size;
+ int16_t next_marked_index;
+ int16_t dummy;
+ VALUE obj;
+};
+
+static struct transient_heap global_transient_heap;
+
+static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj);
+static const void *transient_heap_ptr(VALUE obj, int error);
+static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr);
+
+#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
+
+static void
+transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)
+{
+ int i=0, n=0;
+ struct transient_alloc_header *header = NULL;
+
+ while (i<block->info.index) {
+ header = (void *)&block->buff[i];
+ fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, header, header->size, header->next_marked_index, rb_obj_info(header->obj));
+ i += header->size;
+ n++;
+ }
+}
+
+static void
+transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
+{
+ while (block) {
+ fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
+ type_str, block, block->info.index, block->info.objects, block->info.last_marked_index, block->info.next_block);
+
+ transient_heap_block_dump(theap, block);
+ block = block->info.next_block;
+ }
+}
+
+static void
+transient_heap_dump(struct transient_heap* theap)
+{
+ fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
+ transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");
+ transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");
+ transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");
+}
+
+/* Debug: dump all tarnsient_heap blocks */
+void
+rb_transient_heap_dump(void)
+{
+ transient_heap_dump(&global_transient_heap);
+}
+
+#if TRANSIENT_HEAP_CHECK_MODE >= 2
+static void
+transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)
+{
+ if (obj != Qundef) {
+ const void *ptr = transient_heap_ptr(obj, FALSE);
+ TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr));
+ }
+}
+
+static int
+transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ int i=0, n=0;
+ struct transient_alloc_header *header;
+
+ while (i<block->info.index) {
+ header = (void *)&block->buff[i];
+ TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);
+ transient_heap_ptr_check(theap, header->obj);
+ n ++;
+ i += header->size;
+ }
+ TH_ASSERT(block->info.objects == n);
+
+ return n;
+}
+
+static int
+transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr)
+{
+ int n = 0;
+ struct transient_heap_block *block = blocks;
+ while (block) {
+ n += transient_heap_block_verify(theap, block);
+ *block_num_ptr += 1;
+ block = block->info.next_block;
+ }
+
+ return n;
+}
+#endif
+
+static void
+transient_heap_verify(struct transient_heap *theap)
+{
+#if TRANSIENT_HEAP_CHECK_MODE >= 2
+ int n=0, block_num=0;
+
+ n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
+ n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
+
+ TH_ASSERT(n == theap->total_objects);
+ TH_ASSERT(n >= theap->total_marked_objects);
+ TH_ASSERT(block_num == theap->total_blocks);
+#endif
+}
+
+/* Debug: check assertions for all transient_heap blocks */
+void
+rb_transient_heap_verify(void)
+{
+ transient_heap_verify(&global_transient_heap);
+}
+
+static struct transient_heap*
+transient_heap_get(void)
+{
+ struct transient_heap* theap = &global_transient_heap;
+ transient_heap_verify(theap);
+ return theap;
+}
+
+static void
+reset_block(struct transient_heap_block *block)
+{
+ block->info.size = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header);
+ block->info.index = 0;
+ block->info.objects = 0;
+ block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
+ block->info.next_block = NULL;
+}
+
+static void
+connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ block->info.next_block = theap->free_blocks;
+ theap->free_blocks = block;
+}
+
+static void
+connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ block->info.next_block = theap->using_blocks;
+ theap->using_blocks = block;
+}
+
+#if 0
+static void
+connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)
+{
+ block->info.next_block = theap->marked_blocks;
+ theap->marked_blocks = block;
+}
+#endif
+
+static void
+append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)
+{
+ if (theap->marked_blocks) {
+ struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;
+ while (block) {
+ last_block = block;
+ block = block->info.next_block;
+ }
+
+ TH_ASSERT(last_block->info.next_block == NULL);
+ last_block->info.next_block = append_blocks;
+ }
+ else {
+ theap->marked_blocks = append_blocks;
+ }
+}
+
+static struct transient_heap_block *
+transient_heap_block_alloc(struct transient_heap* theap)
+{
+ struct transient_heap_block *block;
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS,
+ -1, 0);
+ if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);
+#else
+ if (theap->arena == NULL) {
+ theap->arena = rb_aligned_malloc(TRANSIENT_HEAP_BLOCK_SIZE, TRANSIENT_HEAP_TOTAL_SIZE);
+ }
+
+ TH_ASSERT(theap->arena_index < TRANSIENT_HEAP_BLOCK_NUM);
+ block = &theap->arena[theap->arena_index++];
+ TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
+#endif
+ reset_block(block);
+
+ TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);
+ if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, block);
+ return block;
+}
+
+
+static struct transient_heap_block *
+transient_heap_allocatable_block(struct transient_heap* theap)
+{
+ struct transient_heap_block *block;
+
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block = transient_heap_block_alloc(theap);
+ theap->total_blocks++;
+#else
+ /* get one block from free_blocks */
+ block = theap->free_blocks;
+ if (block) {
+ theap->free_blocks = block->info.next_block;
+ block->info.next_block = NULL;
+ theap->total_blocks++;
+ }
+#endif
+
+ return block;
+}
+
+static struct transient_alloc_header *
+transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
+{
+ struct transient_heap_block *block = theap->using_blocks;
+
+ while (block) {
+ TH_ASSERT(block->info.size >= block->info.index);
+
+ if (block->info.size - block->info.index >= (int32_t)size) {
+ struct transient_alloc_header *header = (void *)&block->buff[block->info.index];
+ block->info.index += size;
+ block->info.objects++;
+ return header;
+ }
+ else {
+ block = transient_heap_allocatable_block(theap);
+ if (block) connect_to_using_blocks(theap, block);
+ }
+ }
+
+ return NULL;
+}
+
+void *
+rb_transient_heap_alloc(VALUE obj, size_t req_size)
+{
+ struct transient_heap* theap = transient_heap_get();
+ size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
+
+ TH_ASSERT(RB_TYPE_P(obj, T_ARRAY)); /* supported types */
+
+ if (size > TRANSIENT_HEAP_ALLOC_MAX) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj));
+ return NULL;
+ }
+#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
+ else if (RB_OBJ_PROMOTED_RAW(obj)) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj));
+ return NULL;
+ }
+#else
+ else if (RBASIC_CLASS(obj) == 0) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj));
+ return NULL;
+ }
+#endif
+ else {
+ struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);
+ if (header) {
+ void *ptr;
+
+ header->size = size;
+ header->magic = TRANSIENT_HEAP_ALLOC_MAGIC;
+ header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
+ header->obj = obj; /* TODO: can we eliminate it? */
+ ptr = header + 1;
+
+ theap->total_objects++; /* statistics */
+
+#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
+ if (RB_OBJ_PROMOTED_RAW(obj)) {
+ transient_heap_promote_add(theap, obj);
+ }
+#endif
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", header, ptr, (int)size, rb_obj_info(obj));
+
+ RB_DEBUG_COUNTER_INC(theap_alloc);
+ return ptr;
+ }
+ else {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj));
+ RB_DEBUG_COUNTER_INC(theap_alloc_fail);
+ return NULL;
+ }
+ }
+}
+
+void
+Init_TransientHeap(void)
+{
+ int i, block_num;
+ struct transient_heap* theap = transient_heap_get();
+
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block_num = 0;
+#else
+ TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE);
+ block_num = TRANSIENT_HEAP_BLOCK_NUM;
+#endif
+ for (i=0; i<block_num; i++) {
+ connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
+ }
+ theap->using_blocks = transient_heap_allocatable_block(theap);
+
+ theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE;
+ theap->promoted_objects_index = 0;
+ /* should not use ALLOC_N to be free from GC */
+ theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);
+ if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");
+}
+
+static struct transient_heap_block *
+blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)
+{
+ struct transient_heap_block *block = blocks;
+
+ while (block) {
+ if (block->buff <= (char *)header && (char *)header < block->buff + block->info.size) {
+ return block;
+ }
+ block = block->info.next_block;
+ }
+
+ return NULL;
+}
+
+static struct transient_heap_block *
+alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header)
+{
+ struct transient_heap_block *block;
+
+ if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");
+ return block;
+ }
+ else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");
+ return block;
+ }
+ else {
+ return NULL;
+ }
+ return block;
+}
+
+static struct transient_alloc_header *
+ptr_to_alloc_header(const void *ptr)
+{
+ struct transient_alloc_header *header = (void *)ptr;
+ header -= 1;
+ return header;
+}
+
+static int
+transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr)
+{
+ if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) {
+ return TRUE;
+ }
+ else {
+ return FALSE;
+ }
+}
+
+
+int
+rb_transient_heap_managed_ptr_p(const void *ptr)
+{
+ return transient_header_managed_ptr_p(transient_heap_get(), ptr);
+}
+
+static struct transient_heap_block *
+alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)
+{
+ struct transient_heap_block *block;
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ block = alloc_header_to_block_verbose(theap, header);
+ if (block == NULL) {
+ transient_heap_dump(theap);
+ rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);
+ }
+#else
+ block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
+ TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
+#endif
+ return block;
+}
+
+void
+rb_transient_heap_mark(VALUE obj, const void *ptr)
+{
+ struct transient_alloc_header *header = ptr_to_alloc_header(ptr);
+
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);
+
+#if TRANSIENT_HEAP_CHECK_MODE > 0
+ {
+ struct transient_heap* theap = transient_heap_get();
+ TH_ASSERT(theap->status == transient_heap_marking);
+ TH_ASSERT(transient_header_managed_ptr_p(theap, ptr));
+
+ if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
+ transient_heap_dump(theap);
+ rb_bug("rb_transient_heap_mark: magic is broken");
+ }
+ else if (header->obj != obj) {
+ transient_heap_dump(theap);
+ rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
+ rb_obj_info(header->obj), rb_obj_info(obj));
+ }
+ }
+#endif
+
+ if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) {
+ /* already marked */
+ return;
+ }
+ else {
+ struct transient_heap* theap = transient_heap_get();
+ struct transient_heap_block *block = alloc_header_to_block(theap, header);
+ header->next_marked_index = block->info.last_marked_index;
+ block->info.last_marked_index = (int)((char *)header - block->buff);
+ theap->total_marked_objects++;
+
+ transient_heap_verify(theap);
+ }
+}
+
+static const void *
+transient_heap_ptr(VALUE obj, int error)
+{
+ const void *ptr;
+
+ switch (BUILTIN_TYPE(obj)) {
+ case T_ARRAY:
+ if (RARRAY_TRANSIENT_P(obj)) {
+ TH_ASSERT(!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG));
+ ptr = RARRAY(obj)->as.heap.ptr;
+ }
+ else {
+ ptr = NULL;
+ }
+ break;
+ default:
+ if (error) {
+ rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));
+ }
+ else {
+ ptr = NULL;
+ }
+ }
+
+ return ptr;
+}
+
+static void
+transient_heap_promote_add(struct transient_heap* theap, VALUE obj)
+{
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));
+
+ if (TRANSIENT_HEAP_DEBUG_DONT_PROMOTE) {
+ /* duplicate check */
+ int i;
+ for (i=0; i<theap->promoted_objects_index; i++) {
+ if (theap->promoted_objects[i] == obj) return;
+ }
+ }
+
+ if (theap->promoted_objects_size <= theap->promoted_objects_index) {
+ theap->promoted_objects_size *= 2;
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
+ theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));
+ if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");
+ }
+ theap->promoted_objects[theap->promoted_objects_index++] = obj;
+}
+
+void
+rb_transient_heap_promote(VALUE obj)
+{
+ if (transient_heap_ptr(obj, FALSE)) {
+ struct transient_heap* theap = transient_heap_get();
+ transient_heap_promote_add(theap, obj);
+ }
+ else {
+ /* ignore */
+ }
+}
+
+static struct transient_alloc_header *
+alloc_header(struct transient_heap_block* block, int index)
+{
+ return (void *)&block->buff[index];
+}
+
+static void
+transient_heap_reset(void)
+{
+ struct transient_heap* theap = transient_heap_get();
+ struct transient_heap_block* block;
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");
+
+ block = theap->marked_blocks;
+ while (block) {
+ struct transient_heap_block *next_block = block->info.next_block;
+ theap->total_objects -= block->info.objects;
+#if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
+ if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
+ rb_bug("madvise err:%d", errno);
+ }
+ if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
+ rb_bug("mprotect err:%d", errno);
+ }
+#else
+ reset_block(block);
+ connect_to_free_blocks(theap, block);
+#endif
+ theap->total_blocks--;
+ block = next_block;
+ }
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks);
+
+ theap->marked_blocks = NULL;
+ theap->total_marked_objects = 0;
+}
+
+static void
+transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block)
+{
+ int marked_index = block->info.last_marked_index;
+ block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
+
+ while (marked_index >= 0) {
+ struct transient_alloc_header *header = alloc_header(block, marked_index);
+ VALUE obj = header->obj;
+
+ if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", header, rb_obj_info(obj));
+
+ if (obj != Qnil) {
+ RB_DEBUG_COUNTER_INC(theap_evacuate);
+
+ switch (BUILTIN_TYPE(obj)) {
+ case T_ARRAY:
+#if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
+ rb_ary_transient_heap_evacuate(obj, FALSE);
+#else
+ rb_ary_transient_heap_evacuate(obj, TRUE);
+#endif
+ break;
+ default:
+ rb_bug("unsupporeted");
+ }
+ header->obj = Qundef; /* for debug */
+ }
+ marked_index = header->next_marked_index;
+ }
+}
+
+static void
+transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)
+{
+ TH_ASSERT(theap->status != status);
+ theap->status = status;
+}
+
+static void
+transient_heap_evacuate(void *dmy)
+{
+ struct transient_heap* theap = transient_heap_get();
+
+ if (theap->status == transient_heap_marking) {
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
+ }
+ else {
+ VALUE gc_disabled = rb_gc_disable();
+ struct transient_heap_block* block;
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) {
+ int i;
+ fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
+ if (TRANSIENT_HEAP_DEBUG >= 4) {
+ for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i]));
+ }
+ }
+ if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
+
+ TH_ASSERT(theap->status == transient_heap_none);
+ transient_heap_update_status(theap, transient_heap_escaping);
+
+ /* evacuate from marked blocks */
+ block = theap->marked_blocks;
+ while (block) {
+ transient_heap_block_evacuate(theap, block);
+ block = block->info.next_block;
+ }
+
+ /* evacuate from using blocks
+ only affect incremental marking */
+ block = theap->using_blocks;
+ while (block) {
+ transient_heap_block_evacuate(theap, block);
+ block = block->info.next_block;
+ }
+
+ /* all objects in marked_objects are escaped. */
+ transient_heap_reset();
+
+ if (TRANSIENT_HEAP_DEBUG > 0) {
+ fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
+ }
+
+ transient_heap_verify(theap);
+ transient_heap_update_status(theap, transient_heap_none);
+ if (gc_disabled != Qtrue) rb_gc_enable();
+ }
+}
+
+static void
+clear_marked_index(struct transient_heap_block* block)
+{
+ int marked_index = block->info.last_marked_index;
+
+ while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
+ struct transient_alloc_header *header = alloc_header(block, marked_index);
+ TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE);
+ if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", block, marked_index);
+
+ marked_index = header->next_marked_index;
+ header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;
+ }
+
+ block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;
+}
+
+static void
+blocks_clear_marked_index(struct transient_heap_block* block)
+{
+ while (block) {
+ clear_marked_index(block);
+ block = block->info.next_block;
+ }
+}
+
+void
+rb_transient_heap_start_marking(int full_marking)
+{
+ struct transient_heap* theap = transient_heap_get();
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promtoed:%d full_marking:%d\n",
+ theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
+ if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
+
+ blocks_clear_marked_index(theap->marked_blocks);
+ blocks_clear_marked_index(theap->using_blocks);
+
+ if (theap->using_blocks) {
+ if (theap->using_blocks->info.objects > 0) {
+ append_to_marked_blocks(theap, theap->using_blocks);
+ theap->using_blocks = NULL;
+ }
+ else {
+ append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
+ theap->using_blocks->info.next_block = NULL;
+ }
+ }
+
+ if (theap->using_blocks == NULL) {
+ theap->using_blocks = transient_heap_allocatable_block(theap);
+ }
+
+ TH_ASSERT(theap->status == transient_heap_none);
+ transient_heap_update_status(theap, transient_heap_marking);
+ theap->total_marked_objects = 0;
+
+ if (full_marking) {
+ theap->promoted_objects_index = 0;
+ }
+ else { /* mark promoted objects */
+ int i;
+ for (i=0; i<theap->promoted_objects_index; i++) {
+ VALUE obj = theap->promoted_objects[i];
+ const void *ptr = transient_heap_ptr(obj, TRUE);
+ if (ptr) {
+ rb_transient_heap_mark(obj, ptr);
+ }
+ }
+ }
+
+ transient_heap_verify(theap);
+}
+
+void
+rb_transient_heap_finish_marking(void)
+{
+ struct transient_heap* theap = transient_heap_get();
+
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n",
+ theap->total_objects,
+ theap->total_marked_objects);
+ if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
+
+ TH_ASSERT(theap->total_objects >= theap->total_marked_objects);
+
+ TH_ASSERT(theap->status == transient_heap_marking);
+ transient_heap_update_status(theap, transient_heap_none);
+
+ if (theap->total_marked_objects > 0) {
+ if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");
+ rb_postponed_job_register_one(0, transient_heap_evacuate, NULL);
+ }
+ else {
+ transient_heap_reset();
+ }
+
+ transient_heap_verify(theap);
+}