summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-10-30 20:46:24 +0000
committerko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-10-30 20:46:24 +0000
commit90ac549fa6bf6c5df38302bf1198a119f12a60ba (patch)
treec97c36c697f4e3fe63bee613c7d160918f1ad705
parentb51724aac5b8024dd8515534d66b20f71742415a (diff)
introduce TransientHeap. [Bug #14858]
* transient_heap.c, transient_heap.h: implement TransientHeap (theap). theap is designed for Ruby's object system. theap is like Eden heap on generational GC terminology. theap allocation is very fast because it only needs to bump up pointer and deallocation is also fast because we don't do anything. However we need to evacuate (Copy GC terminology) if theap memory is long-lived. Evacuation logic is needed for each type. See [Bug #14858] for details. * array.c: Now, theap for T_ARRAY is supported. ary_heap_alloc() tries to allocate memory area from theap. If this trial sccesses, this array has theap ptr and RARRAY_TRANSIENT_FLAG is turned on. We don't need to free theap ptr. * ruby.h: RARRAY_CONST_PTR() returns malloc'ed memory area. It menas that if ary is allocated at theap, force evacuation to malloc'ed memory. It makes programs slow, but very compatible with current code because theap memory can be evacuated (theap memory will be recycled). If you want to get transient heap ptr, use RARRAY_CONST_PTR_TRANSIENT() instead of RARRAY_CONST_PTR(). If you can't understand when evacuation will occur, use RARRAY_CONST_PTR(). git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65444 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r--array.c420
-rw-r--r--common.mk3
-rw-r--r--compile.c6
-rw-r--r--debug_counter.h5
-rw-r--r--enum.c8
-rw-r--r--gc.c101
-rw-r--r--include/ruby/ruby.h30
-rw-r--r--inits.c1
-rw-r--r--insns.def2
-rw-r--r--internal.h25
-rw-r--r--string.c6
-rw-r--r--test/ruby/test_enum.rb15
-rw-r--r--vm_args.c8
-rw-r--r--vm_eval.c2
-rw-r--r--vm_insnhelper.c2
15 files changed, 500 insertions, 134 deletions
diff --git a/array.c b/array.c
index 83a5ff4..bbe11fe 100644
--- a/array.c
+++ b/array.c
@@ -14,12 +14,14 @@
#include "ruby/encoding.h"
#include "ruby/util.h"
#include "ruby/st.h"
-#include "internal.h"
#include "probes.h"
#include "id.h"
#include "debug_counter.h"
+#include "gc.h"
+#include "transient_heap.h"
+#include "internal.h"
-#ifndef ARRAY_DEBUG
+#if !ARRAY_DEBUG
# define NDEBUG
#endif
#include "ruby_assert.h"
@@ -42,17 +44,21 @@ VALUE rb_cArray;
#define ARY_HEAP_PTR(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.ptr)
#define ARY_HEAP_LEN(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.len)
+#define ARY_HEAP_CAPA(a) (assert(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.aux.capa)
+
#define ARY_EMBED_PTR(a) (assert(ARY_EMBED_P(a)), RARRAY(a)->as.ary)
#define ARY_EMBED_LEN(a) \
(assert(ARY_EMBED_P(a)), \
(long)((RBASIC(a)->flags >> RARRAY_EMBED_LEN_SHIFT) & \
(RARRAY_EMBED_LEN_MASK >> RARRAY_EMBED_LEN_SHIFT)))
-#define ARY_HEAP_SIZE(a) (assert(!ARY_EMBED_P(a)), assert(ARY_OWNS_HEAP_P(a)), RARRAY(a)->as.heap.aux.capa * sizeof(VALUE))
+#define ARY_HEAP_SIZE(a) (assert(!ARY_EMBED_P(a)), assert(ARY_OWNS_HEAP_P(a)), ARY_HEAP_CAPA(a) * sizeof(VALUE))
#define ARY_OWNS_HEAP_P(a) (!FL_TEST((a), ELTS_SHARED|RARRAY_EMBED_FLAG))
#define FL_SET_EMBED(a) do { \
assert(!ARY_SHARED_P(a)); \
FL_SET((a), RARRAY_EMBED_FLAG); \
+ FL_UNSET_RAW((a), RARRAY_TRANSIENT_FLAG); \
+ ary_verify(a); \
} while (0)
#define FL_UNSET_EMBED(ary) FL_UNSET((ary), RARRAY_EMBED_FLAG|RARRAY_EMBED_LEN_MASK)
#define FL_SET_SHARED(ary) do { \
@@ -102,7 +108,7 @@ VALUE rb_cArray;
} while (0)
#define ARY_CAPA(ary) (ARY_EMBED_P(ary) ? RARRAY_EMBED_LEN_MAX : \
- ARY_SHARED_ROOT_P(ary) ? RARRAY_LEN(ary) : RARRAY(ary)->as.heap.aux.capa)
+ ARY_SHARED_ROOT_P(ary) ? RARRAY_LEN(ary) : ARY_HEAP_CAPA(ary))
#define ARY_SET_CAPA(ary, n) do { \
assert(!ARY_EMBED_P(ary)); \
assert(!ARY_SHARED_P(ary)); \
@@ -130,11 +136,82 @@ VALUE rb_cArray;
} while (0)
#define FL_SET_SHARED_ROOT(ary) do { \
assert(!ARY_EMBED_P(ary)); \
+ assert(!RARRAY_TRANSIENT_P(ary)); \
FL_SET((ary), RARRAY_SHARED_ROOT_FLAG); \
} while (0)
#define ARY_SET(a, i, v) RARRAY_ASET((assert(!ARY_SHARED_P(a)), (a)), (i), (v))
+
+#if ARRAY_DEBUG
+#define ary_verify(ary) ary_verify_(ary, __FILE__, __LINE__)
+
+static VALUE
+ary_verify_(VALUE ary, const char *file, int line)
+{
+ assert(RB_TYPE_P(ary, T_ARRAY));
+
+ if (FL_TEST(ary, ELTS_SHARED)) {
+ VALUE root = RARRAY(ary)->as.heap.aux.shared;
+ const VALUE *ptr = ARY_HEAP_PTR(ary);
+ const VALUE *root_ptr = RARRAY_CONST_PTR_TRANSIENT(root);
+ long len = ARY_HEAP_LEN(ary), root_len = RARRAY_LEN(root);
+ assert(FL_TEST(root, RARRAY_SHARED_ROOT_FLAG));
+ assert(root_ptr <= ptr && ptr + len <= root_ptr + root_len);
+ ary_verify(root);
+ }
+ else if (ARY_EMBED_P(ary)) {
+ assert(!RARRAY_TRANSIENT_P(ary));
+ assert(!ARY_SHARED_P(ary));
+ assert(RARRAY_LEN(ary) <= RARRAY_EMBED_LEN_MAX);
+ }
+ else {
+#if 1
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
+ long i, len = RARRAY_LEN(ary);
+ volatile VALUE v;
+ if (len > 1) len = 1; /* check only HEAD */
+ for (i=0; i<len; i++) {
+ v = ptr[i]; /* access check */
+ }
+ v = v;
+#endif
+ }
+
+ if (RARRAY_TRANSIENT_P(ary)) {
+ assert(rb_transient_heap_managed_ptr_p(RARRAY_CONST_PTR_TRANSIENT(ary)));
+ }
+
+ rb_transient_heap_verify();
+
+ return ary;
+}
+
+void
+rb_ary_verify(VALUE ary){
+ ary_verify(ary);
+}
+#else
+#define ary_verify(ary) ((void)0)
+#endif
+
+VALUE *
+rb_ary_ptr_use_start(VALUE ary)
+{
+#if ARRAY_DEBUG
+ FL_SET_RAW(ary, RARRAY_PTR_IN_USE_FLAG);
+#endif
+ return (VALUE *)RARRAY_CONST_PTR_TRANSIENT(ary);
+}
+
+void
+rb_ary_ptr_use_end(VALUE ary)
+{
+#if ARRAY_DEBUG
+ FL_UNSET_RAW(ary, RARRAY_PTR_IN_USE_FLAG);
+#endif
+}
+
void
rb_mem_clear(register VALUE *mem, register long size)
{
@@ -195,49 +272,167 @@ ary_memcpy(VALUE ary, long beg, long argc, const VALUE *argv)
ary_memcpy0(ary, beg, argc, argv, ary);
}
+static VALUE *
+ary_heap_alloc(VALUE ary, size_t capa)
+{
+ VALUE *ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * capa);
+
+ if (ptr != NULL) {
+ FL_SET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+ else {
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ ptr = ALLOC_N(VALUE, capa);
+ }
+
+ return ptr;
+}
+
+static void
+ary_heap_free_ptr(VALUE ary, const VALUE *ptr, long size)
+{
+ if (RARRAY_TRANSIENT_P(ary)) {
+ /* ignore it */
+ }
+ else {
+ ruby_sized_xfree((void *)ptr, size);
+ }
+}
+
+static void
+ary_heap_free(VALUE ary)
+{
+ if (RARRAY_TRANSIENT_P(ary)) {
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+ else {
+ ary_heap_free_ptr(ary, ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));
+ }
+}
+
+static void
+ary_heap_realloc(VALUE ary, size_t new_capa)
+{
+ size_t old_capa = ARY_HEAP_CAPA(ary);
+
+ if (RARRAY_TRANSIENT_P(ary)) {
+ if (new_capa <= old_capa) {
+ /* do nothing */
+ }
+ else {
+ VALUE *new_ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * new_capa);
+
+ if (new_ptr == NULL) {
+ new_ptr = ALLOC_N(VALUE, new_capa);
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+
+ MEMCPY(new_ptr, ARY_HEAP_PTR(ary), VALUE, old_capa);
+ ARY_SET_PTR(ary, new_ptr);
+ }
+ }
+ else {
+ SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, old_capa);
+ }
+ ary_verify(ary);
+}
+
+static inline void
+rb_ary_transient_heap_evacuate_(VALUE ary, int transient, int promote)
+{
+ if (transient) {
+ VALUE *new_ptr;
+ const VALUE *old_ptr = ARY_HEAP_PTR(ary);
+ long capa = ARY_HEAP_CAPA(ary);
+ long len = ARY_HEAP_LEN(ary);
+
+ if (ARY_SHARED_ROOT_P(ary)) {
+ capa = len;
+ }
+
+ assert(ARY_OWNS_HEAP_P(ary));
+ assert(RARRAY_TRANSIENT_P(ary));
+ assert(!ARY_PTR_USING_P(ary));
+
+ if (promote) {
+ new_ptr = ALLOC_N(VALUE, capa);
+ FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);
+ }
+ else {
+ new_ptr = ary_heap_alloc(ary, capa);
+ }
+
+ MEMCPY(new_ptr, old_ptr, VALUE, capa);
+ /* do not use ARY_SET_PTR() because they assert !frozen */
+ RARRAY(ary)->as.heap.ptr = new_ptr;
+ }
+
+ ary_verify(ary);
+}
+
+void
+rb_ary_transient_heap_evacuate(VALUE ary, int promote)
+{
+ rb_ary_transient_heap_evacuate_(ary, RARRAY_TRANSIENT_P(ary), promote);
+}
+
+void
+rb_ary_detransient(VALUE ary)
+{
+ assert(RARRAY_TRANSIENT_P(ary));
+ rb_ary_transient_heap_evacuate_(ary, TRUE, TRUE);
+}
+
static void
ary_resize_capa(VALUE ary, long capacity)
{
assert(RARRAY_LEN(ary) <= capacity);
assert(!OBJ_FROZEN(ary));
assert(!ARY_SHARED_P(ary));
+
if (capacity > RARRAY_EMBED_LEN_MAX) {
if (ARY_EMBED_P(ary)) {
long len = ARY_EMBED_LEN(ary);
- VALUE *ptr = ALLOC_N(VALUE, (capacity));
+ VALUE *ptr = ary_heap_alloc(ary, capacity);
+
MEMCPY(ptr, ARY_EMBED_PTR(ary), VALUE, len);
FL_UNSET_EMBED(ary);
ARY_SET_PTR(ary, ptr);
ARY_SET_HEAP_LEN(ary, len);
}
else {
- SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, RARRAY(ary)->as.heap.aux.capa);
+ ary_heap_realloc(ary, capacity);
}
- ARY_SET_CAPA(ary, (capacity));
+ ARY_SET_CAPA(ary, capacity);
}
else {
if (!ARY_EMBED_P(ary)) {
- long len = RARRAY_LEN(ary);
- const VALUE *ptr = RARRAY_CONST_PTR(ary);
+ long len = ARY_HEAP_LEN(ary);
+ long old_capa = ARY_HEAP_CAPA(ary);
+ const VALUE *ptr = ARY_HEAP_PTR(ary);
- if (len > capacity) len = capacity;
+ if (len > capacity) len = capacity;
MEMCPY((VALUE *)RARRAY(ary)->as.ary, ptr, VALUE, len);
+ ary_heap_free_ptr(ary, ptr, old_capa);
+
FL_SET_EMBED(ary);
ARY_SET_LEN(ary, len);
- ruby_sized_xfree((VALUE *)ptr, RARRAY(ary)->as.heap.aux.capa);
}
}
+
+ ary_verify(ary);
}
static inline void
ary_shrink_capa(VALUE ary)
{
long capacity = ARY_HEAP_LEN(ary);
- long old_capa = RARRAY(ary)->as.heap.aux.capa;
+ long old_capa = ARY_HEAP_CAPA(ary);
assert(!ARY_SHARED_P(ary));
assert(old_capa >= capacity);
- if (old_capa > capacity)
- SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, capacity, old_capa);
+ if (old_capa > capacity) ary_heap_realloc(ary, capacity);
+
+ ary_verify(ary);
}
static void
@@ -253,6 +448,8 @@ ary_double_capa(VALUE ary, long min)
}
new_capa += min;
ary_resize_capa(ary, new_capa);
+
+ ary_verify(ary);
}
static void
@@ -308,6 +505,7 @@ static inline void
rb_ary_modify_check(VALUE ary)
{
rb_check_frozen(ary);
+ ary_verify(ary);
}
void
@@ -317,6 +515,9 @@ rb_ary_modify(VALUE ary)
if (ARY_SHARED_P(ary)) {
long shared_len, len = RARRAY_LEN(ary);
VALUE shared = ARY_SHARED(ary);
+
+ ary_verify(shared);
+
if (len <= RARRAY_EMBED_LEN_MAX) {
const VALUE *ptr = ARY_HEAP_PTR(ary);
FL_UNSET_SHARED(ary);
@@ -326,9 +527,9 @@ rb_ary_modify(VALUE ary)
ARY_SET_EMBED_LEN(ary, len);
}
else if (ARY_SHARED_OCCUPIED(shared) && len > ((shared_len = RARRAY_LEN(shared))>>1)) {
- long shift = RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared);
+ long shift = RARRAY_CONST_PTR_TRANSIENT(ary) - RARRAY_CONST_PTR_TRANSIENT(shared);
FL_UNSET_SHARED(ary);
- ARY_SET_PTR(ary, RARRAY_CONST_PTR(shared));
+ ARY_SET_PTR(ary, RARRAY_CONST_PTR_TRANSIENT(shared));
ARY_SET_CAPA(ary, shared_len);
RARRAY_PTR_USE(ary, ptr, {
MEMMOVE(ptr, ptr+shift, VALUE, len);
@@ -337,8 +538,8 @@ rb_ary_modify(VALUE ary)
rb_ary_decrement_share(shared);
}
else {
- VALUE *ptr = ALLOC_N(VALUE, len);
- MEMCPY(ptr, RARRAY_CONST_PTR(ary), VALUE, len);
+ VALUE *ptr = ary_heap_alloc(ary, len);
+ MEMCPY(ptr, ARY_HEAP_PTR(ary), VALUE, len);
rb_ary_unshare(ary);
ARY_SET_CAPA(ary, len);
ARY_SET_PTR(ary, ptr);
@@ -346,6 +547,7 @@ rb_ary_modify(VALUE ary)
rb_gc_writebarrier_remember(ary);
}
+ ary_verify(ary);
}
static VALUE
@@ -362,9 +564,12 @@ ary_ensure_room_for_push(VALUE ary, long add_len)
if (new_len > RARRAY_EMBED_LEN_MAX) {
VALUE shared = ARY_SHARED(ary);
if (ARY_SHARED_OCCUPIED(shared)) {
- if (RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared) + new_len <= RARRAY_LEN(shared)) {
+ if (ARY_HEAP_PTR(ary) - RARRAY_CONST_PTR_TRANSIENT(shared) + new_len <= RARRAY_LEN(shared)) {
rb_ary_modify_check(ary);
- return shared;
+
+ ary_verify(ary);
+ ary_verify(shared);
+ return shared;
}
else {
/* if array is shared, then it is likely it participate in push/shift pattern */
@@ -373,11 +578,13 @@ ary_ensure_room_for_push(VALUE ary, long add_len)
if (new_len > capa - (capa >> 6)) {
ary_double_capa(ary, new_len);
}
+ ary_verify(ary);
return ary;
}
}
}
- rb_ary_modify(ary);
+ ary_verify(ary);
+ rb_ary_modify(ary);
}
else {
rb_ary_modify_check(ary);
@@ -387,6 +594,7 @@ ary_ensure_room_for_push(VALUE ary, long add_len)
ary_double_capa(ary, new_len);
}
+ ary_verify(ary);
return ary;
}
@@ -459,7 +667,7 @@ ary_new(VALUE klass, long capa)
ary = ary_alloc(klass);
if (capa > RARRAY_EMBED_LEN_MAX) {
- ptr = ALLOC_N(VALUE, capa);
+ ptr = ary_heap_alloc(ary, capa);
FL_UNSET_EMBED(ary);
ARY_SET_PTR(ary, ptr);
ARY_SET_CAPA(ary, capa);
@@ -523,7 +731,9 @@ rb_ary_new_from_values(long n, const VALUE *elts)
VALUE
rb_ary_tmp_new(long capa)
{
- return ary_new(0, capa);
+ VALUE ary = ary_new(0, capa);
+ rb_ary_transient_heap_evacuate(ary, TRUE);
+ return ary;
}
VALUE
@@ -532,6 +742,7 @@ rb_ary_tmp_new_fill(long capa)
VALUE ary = ary_new(0, capa);
ary_memfill(ary, 0, capa, Qnil);
ARY_SET_LEN(ary, capa);
+ rb_ary_transient_heap_evacuate(ary, TRUE);
return ary;
}
@@ -539,8 +750,13 @@ void
rb_ary_free(VALUE ary)
{
if (ARY_OWNS_HEAP_P(ary)) {
- RB_DEBUG_COUNTER_INC(obj_ary_ptr);
- ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));
+ if (RARRAY_TRANSIENT_P(ary)) {
+ RB_DEBUG_COUNTER_INC(obj_ary_transient);
+ }
+ else {
+ RB_DEBUG_COUNTER_INC(obj_ary_ptr);
+ ary_heap_free(ary);
+ }
}
else {
RB_DEBUG_COUNTER_INC(obj_ary_embed);
@@ -563,13 +779,15 @@ ary_discard(VALUE ary)
{
rb_ary_free(ary);
RBASIC(ary)->flags |= RARRAY_EMBED_FLAG;
- RBASIC(ary)->flags &= ~RARRAY_EMBED_LEN_MASK;
+ RBASIC(ary)->flags &= ~(RARRAY_EMBED_LEN_MASK | RARRAY_TRANSIENT_FLAG);
}
static VALUE
ary_make_shared(VALUE ary)
{
assert(!ARY_EMBED_P(ary));
+ ary_verify(ary);
+
if (ARY_SHARED_P(ary)) {
return ARY_SHARED(ary);
}
@@ -577,6 +795,7 @@ ary_make_shared(VALUE ary)
return ary;
}
else if (OBJ_FROZEN(ary)) {
+ rb_ary_transient_heap_evacuate(ary, TRUE);
ary_shrink_capa(ary);
FL_SET_SHARED_ROOT(ary);
ARY_SET_SHARED_NUM(ary, 1);
@@ -584,18 +803,25 @@ ary_make_shared(VALUE ary)
}
else {
long capa = ARY_CAPA(ary), len = RARRAY_LEN(ary);
+ const VALUE *ptr;
NEWOBJ_OF(shared, struct RArray, 0, T_ARRAY | (RGENGC_WB_PROTECTED_ARRAY ? FL_WB_PROTECTED : 0));
- FL_UNSET_EMBED(shared);
+ rb_ary_transient_heap_evacuate(ary, TRUE);
+ ptr = ARY_HEAP_PTR(ary);
+
+ FL_UNSET_EMBED(shared);
ARY_SET_LEN((VALUE)shared, capa);
- ARY_SET_PTR((VALUE)shared, RARRAY_CONST_PTR(ary));
- ary_mem_clear((VALUE)shared, len, capa - len);
+ ARY_SET_PTR((VALUE)shared, ptr);
+ ary_mem_clear((VALUE)shared, len, capa - len);
FL_SET_SHARED_ROOT(shared);
ARY_SET_SHARED_NUM((VALUE)shared, 1);
FL_SET_SHARED(ary);
ARY_SET_SHARED(ary, (VALUE)shared);
OBJ_FREEZE(shared);
- return (VALUE)shared;
+
+ ary_verify((VALUE)shared);
+ ary_verify(ary);
+ return (VALUE)shared;
}
}
@@ -606,7 +832,7 @@ ary_make_substitution(VALUE ary)
if (len <= RARRAY_EMBED_LEN_MAX) {
VALUE subst = rb_ary_new2(len);
- ary_memcpy(subst, 0, len, RARRAY_CONST_PTR(ary));
+ ary_memcpy(subst, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary));
ARY_SET_EMBED_LEN(subst, len);
return subst;
}
@@ -729,8 +955,8 @@ rb_ary_initialize(int argc, VALUE *argv, VALUE ary)
rb_ary_modify(ary);
if (argc == 0) {
- if (ARY_OWNS_HEAP_P(ary) && RARRAY_CONST_PTR(ary) != 0) {
- ruby_sized_xfree((void *)RARRAY_CONST_PTR(ary), ARY_HEAP_SIZE(ary));
+ if (ARY_OWNS_HEAP_P(ary) && ARY_HEAP_PTR(ary) != NULL) {
+ ary_heap_free(ary);
}
rb_ary_unshare_safe(ary);
FL_SET_EMBED(ary);
@@ -837,7 +1063,7 @@ ary_make_partial(VALUE ary, VALUE klass, long offset, long len)
if (len <= RARRAY_EMBED_LEN_MAX) {
VALUE result = ary_alloc(klass);
- ary_memcpy(result, 0, len, RARRAY_CONST_PTR(ary) + offset);
+ ary_memcpy(result, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary) + offset);
ARY_SET_EMBED_LEN(result, len);
return result;
}
@@ -846,12 +1072,15 @@ ary_make_partial(VALUE ary, VALUE klass, long offset, long len)
FL_UNSET_EMBED(result);
shared = ary_make_shared(ary);
- ARY_SET_PTR(result, RARRAY_CONST_PTR(ary));
+ ARY_SET_PTR(result, RARRAY_CONST_PTR_TRANSIENT(ary));
ARY_SET_LEN(result, RARRAY_LEN(ary));
rb_ary_set_shared(result, shared);
ARY_INCREASE_PTR(result, offset);
ARY_SET_LEN(result, len);
+
+ ary_verify(shared);
+ ary_verify(result);
return result;
}
}
@@ -910,12 +1139,13 @@ ary_take_first_or_last(int argc, const VALUE *argv, VALUE ary, enum ary_take_pos
VALUE
rb_ary_push(VALUE ary, VALUE item)
{
- long idx = RARRAY_LEN(ary);
+ long idx = RARRAY_LEN((ary_verify(ary), ary));
VALUE target_ary = ary_ensure_room_for_push(ary, 1);
RARRAY_PTR_USE(ary, ptr, {
RB_OBJ_WRITE(target_ary, &ptr[idx], item);
});
ARY_SET_LEN(ary, idx + 1);
+ ary_verify(ary);
return ary;
}
@@ -967,6 +1197,7 @@ rb_ary_pop(VALUE ary)
}
--n;
ARY_SET_LEN(ary, n);
+ ary_verify(ary);
return RARRAY_AREF(ary, n);
}
@@ -1000,6 +1231,7 @@ rb_ary_pop_m(int argc, VALUE *argv, VALUE ary)
rb_ary_modify_check(ary);
result = ary_take_first_or_last(argc, argv, ary, ARY_TAKE_LAST);
ARY_INCREASE_LEN(ary, -RARRAY_LEN(result));
+ ary_verify(ary);
return result;
}
@@ -1018,6 +1250,7 @@ rb_ary_shift(VALUE ary)
MEMMOVE(ptr, ptr+1, VALUE, len-1);
}); /* WB: no new reference */
ARY_INCREASE_LEN(ary, -1);
+ ary_verify(ary);
return top;
}
assert(!ARY_EMBED_P(ary)); /* ARY_EMBED_LEN_MAX < ARY_DEFAULT_SIZE */
@@ -1031,6 +1264,8 @@ rb_ary_shift(VALUE ary)
ARY_INCREASE_PTR(ary, 1); /* shift ptr */
ARY_INCREASE_LEN(ary, -1);
+ ary_verify(ary);
+
return top;
}
@@ -1101,6 +1336,7 @@ rb_ary_behead(VALUE ary, long n)
}
ARY_INCREASE_LEN(ary, -n);
+ ary_verify(ary);
return ary;
}
@@ -1120,8 +1356,8 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
VALUE shared = ARY_SHARED(ary);
capa = RARRAY_LEN(shared);
if (ARY_SHARED_OCCUPIED(shared) && capa > new_len) {
- head = RARRAY_CONST_PTR(ary);
- sharedp = RARRAY_CONST_PTR(shared);
+ head = RARRAY_CONST_PTR_TRANSIENT(ary);
+ sharedp = RARRAY_CONST_PTR_TRANSIENT(shared);
goto makeroom_if_need;
}
}
@@ -1134,11 +1370,13 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
/* use shared array for big "queues" */
if (new_len > ARY_DEFAULT_SIZE * 4) {
- /* make a room for unshifted items */
+ ary_verify(ary);
+
+ /* make a room for unshifted items */
capa = ARY_CAPA(ary);
ary_make_shared(ary);
- head = sharedp = RARRAY_CONST_PTR(ary);
+ head = sharedp = RARRAY_CONST_PTR_TRANSIENT(ary);
goto makeroom;
makeroom_if_need:
if (head - sharedp < argc) {
@@ -1151,6 +1389,8 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
}
ARY_SET_PTR(ary, head - argc);
assert(ARY_SHARED_OCCUPIED(ARY_SHARED(ary)));
+
+ ary_verify(ary);
return ARY_SHARED(ary);
}
else {
@@ -1159,6 +1399,7 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
MEMMOVE(ptr + argc, ptr, VALUE, len);
});
+ ary_verify(ary);
return ary;
}
}
@@ -1576,7 +1817,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen)
}
{
- const VALUE *optr = RARRAY_CONST_PTR(ary);
+ const VALUE *optr = RARRAY_CONST_PTR_TRANSIENT(ary);
rofs = (rptr >= optr && rptr < optr + olen) ? rptr - optr : -1;
}
@@ -1589,7 +1830,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen)
len = beg + rlen;
ary_mem_clear(ary, olen, beg - olen);
if (rlen > 0) {
- if (rofs != -1) rptr = RARRAY_CONST_PTR(ary) + rofs;
+ if (rofs != -1) rptr = RARRAY_CONST_PTR_TRANSIENT(ary) + rofs;
ary_memcpy0(ary, beg, rlen, rptr, target_ary);
}
ARY_SET_LEN(ary, len);
@@ -1613,7 +1854,7 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen)
ARY_SET_LEN(ary, alen);
}
if (rlen > 0) {
- if (rofs != -1) rptr = RARRAY_CONST_PTR(ary) + rofs;
+ if (rofs != -1) rptr = RARRAY_CONST_PTR_TRANSIENT(ary) + rofs;
/* give up wb-protected ary */
MEMMOVE(RARRAY_PTR(ary) + beg, rptr, VALUE, rlen);
}
@@ -1673,11 +1914,12 @@ rb_ary_resize(VALUE ary, long len)
}
else {
if (olen > len + ARY_DEFAULT_SIZE) {
- SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, len, RARRAY(ary)->as.heap.aux.capa);
+ ary_heap_realloc(ary, len);
ARY_SET_CAPA(ary, len);
}
ARY_SET_HEAP_LEN(ary, len);
}
+ ary_verify(ary);
return ary;
}
@@ -1738,7 +1980,7 @@ rb_ary_aset(int argc, VALUE *argv, VALUE ary)
/* check if idx is Range */
range:
rpl = rb_ary_to_ary(argv[argc-1]);
- rb_ary_splice(ary, beg, len, RARRAY_CONST_PTR(rpl), RARRAY_LEN(rpl));
+ rb_ary_splice(ary, beg, len, RARRAY_CONST_PTR_TRANSIENT(rpl), RARRAY_LEN(rpl));
RB_GC_GUARD(rpl);
return argv[argc-1];
}
@@ -1820,7 +2062,7 @@ VALUE
rb_ary_each(VALUE ary)
{
long i;
-
+ ary_verify(ary);
RETURN_SIZED_ENUMERATOR(ary, 0, 0, ary_enum_length);
for (i=0; i<RARRAY_LEN(ary); i++) {
rb_yield(RARRAY_AREF(ary, i));
@@ -1930,15 +2172,18 @@ rb_ary_dup(VALUE ary)
{
long len = RARRAY_LEN(ary);
VALUE dup = rb_ary_new2(len);
- ary_memcpy(dup, 0, len, RARRAY_CONST_PTR(ary));
+ ary_memcpy(dup, 0, len, RARRAY_CONST_PTR_TRANSIENT(ary));
ARY_SET_LEN(dup, len);
+
+ ary_verify(ary);
+ ary_verify(dup);
return dup;
}
VALUE
rb_ary_resurrect(VALUE ary)
{
- return rb_ary_new4(RARRAY_LEN(ary), RARRAY_CONST_PTR(ary));
+ return rb_ary_new4(RARRAY_LEN(ary), RARRAY_CONST_PTR_TRANSIENT(ary));
}
extern VALUE rb_output_fs;
@@ -2279,8 +2524,8 @@ rb_ary_reverse_m(VALUE ary)
VALUE dup = rb_ary_new2(len);
if (len > 0) {
- const VALUE *p1 = RARRAY_CONST_PTR(ary);
- VALUE *p2 = (VALUE *)RARRAY_CONST_PTR(dup) + len - 1;
+ const VALUE *p1 = RARRAY_CONST_PTR_TRANSIENT(ary);
+ VALUE *p2 = (VALUE *)RARRAY_CONST_PTR_TRANSIENT(dup) + len - 1;
do *p2-- = *p1++; while (--len > 0);
}
ARY_SET_LEN(dup, RARRAY_LEN(ary));
@@ -2382,7 +2627,7 @@ rb_ary_rotate_m(int argc, VALUE *argv, VALUE ary)
rotated = rb_ary_new2(len);
if (len > 0) {
cnt = rotate_count(cnt, len);
- ptr = RARRAY_CONST_PTR(ary);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
len -= cnt;
ary_memcpy(rotated, 0, len, ptr + cnt);
ary_memcpy(rotated, len, cnt, ptr);
@@ -2482,7 +2727,6 @@ rb_ary_sort_bang(VALUE ary)
VALUE tmp = ary_make_substitution(ary); /* only ary refers tmp */
struct ary_sort_data data;
long len = RARRAY_LEN(ary);
-
RBASIC_CLEAR_CLASS(tmp);
data.ary = tmp;
data.cmp_opt.opt_methods = 0;
@@ -2515,21 +2759,22 @@ rb_ary_sort_bang(VALUE ary)
rb_ary_unshare(ary);
}
else {
- ruby_sized_xfree((void *)ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));
+ ary_heap_free(ary);
}
- ARY_SET_PTR(ary, RARRAY_CONST_PTR(tmp));
+ ARY_SET_PTR(ary, ARY_HEAP_PTR(tmp));
ARY_SET_HEAP_LEN(ary, len);
- ARY_SET_CAPA(ary, RARRAY_LEN(tmp));
+ ARY_SET_CAPA(ary, ARY_HEAP_LEN(tmp));
}
/* tmp was lost ownership for the ptr */
FL_UNSET(tmp, FL_FREEZE);
FL_SET_EMBED(tmp);
ARY_SET_EMBED_LEN(tmp, 0);
FL_SET(tmp, FL_FREEZE);
- }
+ }
/* tmp will be GC'ed. */
RBASIC_SET_CLASS_RAW(tmp, rb_cArray); /* rb_cArray must be marked */
}
+ ary_verify(ary);
return ary;
}
@@ -2833,7 +3078,7 @@ append_values_at_single(VALUE result, VALUE ary, long olen, VALUE idx)
/* check if idx is Range */
else if (rb_range_beg_len(idx, &beg, &len, olen, 1)) {
if (len > 0) {
- const VALUE *const src = RARRAY_CONST_PTR(ary);
+ const VALUE *const src = RARRAY_CONST_PTR_TRANSIENT(ary);
const long end = beg + len;
const long prevlen = RARRAY_LEN(result);
if (beg < olen) {
@@ -3075,6 +3320,7 @@ rb_ary_delete(VALUE ary, VALUE item)
ary_resize_smaller(ary, i2);
+ ary_verify(ary);
return v;
}
@@ -3119,7 +3365,7 @@ rb_ary_delete_at(VALUE ary, long pos)
MEMMOVE(ptr+pos, ptr+pos+1, VALUE, len-pos-1);
});
ARY_INCREASE_LEN(ary, -1);
-
+ ary_verify(ary);
return del;
}
@@ -3187,7 +3433,7 @@ rb_ary_slice_bang(int argc, VALUE *argv, VALUE ary)
len = orig_len - pos;
}
if (len == 0) return rb_ary_new2(0);
- arg2 = rb_ary_new4(len, RARRAY_CONST_PTR(ary)+pos);
+ arg2 = rb_ary_new4(len, RARRAY_CONST_PTR_TRANSIENT(ary)+pos);
RBASIC_SET_CLASS(arg2, rb_obj_class(ary));
rb_ary_splice(ary, pos, len, 0, 0);
return arg2;
@@ -3223,7 +3469,8 @@ ary_reject(VALUE orig, VALUE result)
for (i = 0; i < RARRAY_LEN(orig); i++) {
VALUE v = RARRAY_AREF(orig, i);
- if (!RTEST(rb_yield(v))) {
+
+ if (!RTEST(rb_yield(v))) {
rb_ary_push(result, v);
}
}
@@ -3252,7 +3499,6 @@ static VALUE
ary_reject_bang(VALUE ary)
{
struct select_bang_arg args;
-
rb_ary_modify_check(ary);
args.ary = ary;
args.len[0] = args.len[1] = 0;
@@ -3326,6 +3572,7 @@ rb_ary_reject(VALUE ary)
static VALUE
rb_ary_delete_if(VALUE ary)
{
+ ary_verify(ary);
RETURN_SIZED_ENUMERATOR(ary, 0, 0, ary_enum_length);
ary_reject_bang(ary);
return ary;
@@ -3504,14 +3751,14 @@ rb_ary_replace(VALUE copy, VALUE orig)
VALUE shared = 0;
if (ARY_OWNS_HEAP_P(copy)) {
- RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy)));
+ ary_heap_free(copy);
}
else if (ARY_SHARED_P(copy)) {
shared = ARY_SHARED(copy);
FL_UNSET_SHARED(copy);
}
FL_SET_EMBED(copy);
- ary_memcpy(copy, 0, RARRAY_LEN(orig), RARRAY_CONST_PTR(orig));
+ ary_memcpy(copy, 0, RARRAY_LEN(orig), RARRAY_CONST_PTR_TRANSIENT(orig));
if (shared) {
rb_ary_decrement_share(shared);
}
@@ -3520,16 +3767,17 @@ rb_ary_replace(VALUE copy, VALUE orig)
else {
VALUE shared = ary_make_shared(orig);
if (ARY_OWNS_HEAP_P(copy)) {
- RARRAY_PTR_USE(copy, ptr, ruby_sized_xfree(ptr, ARY_HEAP_SIZE(copy)));
+ ary_heap_free(copy);
}
else {
rb_ary_unshare_safe(copy);
}
FL_UNSET_EMBED(copy);
- ARY_SET_PTR(copy, RARRAY_CONST_PTR(orig));
- ARY_SET_LEN(copy, RARRAY_LEN(orig));
+ ARY_SET_PTR(copy, ARY_HEAP_PTR(orig));
+ ARY_SET_LEN(copy, ARY_HEAP_LEN(orig));
rb_ary_set_shared(copy, shared);
}
+ ary_verify(copy);
return copy;
}
@@ -3547,16 +3795,20 @@ VALUE
rb_ary_clear(VALUE ary)
{
rb_ary_modify_check(ary);
- ARY_SET_LEN(ary, 0);
if (ARY_SHARED_P(ary)) {
if (!ARY_EMBED_P(ary)) {
rb_ary_unshare(ary);
FL_SET_EMBED(ary);
+ ARY_SET_EMBED_LEN(ary, 0);
}
}
- else if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) {
- ary_resize_capa(ary, ARY_DEFAULT_SIZE * 2);
+ else {
+ ARY_SET_LEN(ary, 0);
+ if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) {
+ ary_resize_capa(ary, ARY_DEFAULT_SIZE * 2);
+ }
}
+ ary_verify(ary);
return ary;
}
@@ -3689,8 +3941,8 @@ rb_ary_plus(VALUE x, VALUE y)
len = xlen + ylen;
z = rb_ary_new2(len);
- ary_memcpy(z, 0, xlen, RARRAY_CONST_PTR(x));
- ary_memcpy(z, xlen, ylen, RARRAY_CONST_PTR(y));
+ ary_memcpy(z, 0, xlen, RARRAY_CONST_PTR_TRANSIENT(x));
+ ary_memcpy(z, xlen, ylen, RARRAY_CONST_PTR_TRANSIENT(y));
ARY_SET_LEN(z, len);
return z;
}
@@ -3700,7 +3952,7 @@ ary_append(VALUE x, VALUE y)
{
long n = RARRAY_LEN(y);
if (n > 0) {
- rb_ary_splice(x, RARRAY_LEN(x), 0, RARRAY_CONST_PTR(y), n);
+ rb_ary_splice(x, RARRAY_LEN(x), 0, RARRAY_CONST_PTR_TRANSIENT(y), n);
}
return x;
}
@@ -3742,6 +3994,7 @@ rb_ary_concat_multi(int argc, VALUE *argv, VALUE ary)
ary_append(ary, args);
}
+ ary_verify(ary);
return ary;
}
@@ -3796,16 +4049,16 @@ rb_ary_times(VALUE ary, VALUE times)
ary2 = ary_new(rb_obj_class(ary), len);
ARY_SET_LEN(ary2, len);
- ptr = RARRAY_CONST_PTR(ary);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
t = RARRAY_LEN(ary);
if (0 < t) {
ary_memcpy(ary2, 0, t, ptr);
while (t <= len/2) {
- ary_memcpy(ary2, t, t, RARRAY_CONST_PTR(ary2));
+ ary_memcpy(ary2, t, t, RARRAY_CONST_PTR_TRANSIENT(ary2));
t *= 2;
}
if (t < len) {
- ary_memcpy(ary2, t, len-t, RARRAY_CONST_PTR(ary2));
+ ary_memcpy(ary2, t, len-t, RARRAY_CONST_PTR_TRANSIENT(ary2));
}
}
out:
@@ -3891,6 +4144,7 @@ recursive_equal(VALUE ary1, VALUE ary2, int recur)
if (recur) return Qtrue; /* Subtle! */
+ /* rb_equal() can evacuate ptrs */
p1 = RARRAY_CONST_PTR(ary1);
p2 = RARRAY_CONST_PTR(ary2);
len1 = RARRAY_LEN(ary1);
@@ -3903,8 +4157,8 @@ recursive_equal(VALUE ary1, VALUE ary2, int recur)
return Qfalse;
if (len1 < i)
return Qtrue;
- p1 = RARRAY_CONST_PTR(ary1) + i;
- p2 = RARRAY_CONST_PTR(ary2) + i;
+ p1 = RARRAY_CONST_PTR(ary1) + i;
+ p2 = RARRAY_CONST_PTR(ary2) + i;
}
else {
return Qfalse;
@@ -3941,7 +4195,7 @@ rb_ary_equal(VALUE ary1, VALUE ary2)
return rb_equal(ary2, ary1);
}
if (RARRAY_LEN(ary1) != RARRAY_LEN(ary2)) return Qfalse;
- if (RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2)) return Qtrue;
+ if (RARRAY_CONST_PTR_TRANSIENT(ary1) == RARRAY_CONST_PTR_TRANSIENT(ary2)) return Qtrue;
return rb_exec_recursive_paired(recursive_equal, ary1, ary2, ary2);
}
@@ -3972,7 +4226,7 @@ rb_ary_eql(VALUE ary1, VALUE ary2)
if (ary1 == ary2) return Qtrue;
if (!RB_TYPE_P(ary2, T_ARRAY)) return Qfalse;
if (RARRAY_LEN(ary1) != RARRAY_LEN(ary2)) return Qfalse;
- if (RARRAY_CONST_PTR(ary1) == RARRAY_CONST_PTR(ary2)) return Qtrue;
+ if (RARRAY_CONST_PTR_TRANSIENT(ary1) == RARRAY_CONST_PTR_TRANSIENT(ary2)) return Qtrue;
return rb_exec_recursive_paired(recursive_eql, ary1, ary2, ary2);
}
@@ -4679,14 +4933,14 @@ rb_ary_compact_bang(VALUE ary)
long n;
rb_ary_modify(ary);
- p = t = (VALUE *)RARRAY_CONST_PTR(ary); /* WB: no new reference */
+ p = t = (VALUE *)RARRAY_CONST_PTR_TRANSIENT(ary); /* WB: no new reference */
end = p + RARRAY_LEN(ary);
while (t < end) {
if (NIL_P(*t)) t++;
else *p++ = *t++;
}
- n = p - RARRAY_CONST_PTR(ary);
+ n = p - RARRAY_CONST_PTR_TRANSIENT(ary);
if (RARRAY_LEN(ary) == n) {
return Qnil;
}
@@ -4949,8 +5203,8 @@ rb_ary_shuffle_bang(int argc, VALUE *argv, VALUE ary)
while (i) {
long j = RAND_UPTO(i);
VALUE tmp;
- if (len != RARRAY_LEN(ary) || ptr != RARRAY_CONST_PTR(ary)) {
- rb_raise(rb_eRuntimeError, "modified during shuffle");
+ if (len != RARRAY_LEN(ary) || ptr != RARRAY_CONST_PTR_TRANSIENT(ary)) {
+ rb_raise(rb_eRuntimeError, "modified during shuffle");
}
tmp = ptr[--i];
ptr[i] = ptr[j];
diff --git a/common.mk b/common.mk
index 42584f6..9a8456f 100644
--- a/common.mk
+++ b/common.mk
@@ -130,6 +130,7 @@ COMMONOBJS = array.$(OBJEXT) \
thread.$(OBJEXT) \
time.$(OBJEXT) \
transcode.$(OBJEXT) \
+ transient_heap.$(OBJEXT) \
util.$(OBJEXT) \
variable.$(OBJEXT) \
version.$(OBJEXT) \
@@ -2898,6 +2899,8 @@ transcode.$(OBJEXT): {$(VPATH)}st.h
transcode.$(OBJEXT): {$(VPATH)}subst.h
transcode.$(OBJEXT): {$(VPATH)}transcode.c
transcode.$(OBJEXT): {$(VPATH)}transcode_data.h
+transient_heap.$(OBJEXT): {$(VPATH)}debug_counter.h
+transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.c
util.$(OBJEXT): $(hdrdir)/ruby/ruby.h
util.$(OBJEXT): $(top_srcdir)/include/ruby.h
util.$(OBJEXT): {$(VPATH)}config.h
diff --git a/compile.c b/compile.c
index ba6924e..58575ec 100644
--- a/compile.c
+++ b/compile.c
@@ -1649,7 +1649,7 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons
opt_table = ALLOC_N(VALUE, i+1);
- MEMCPY(opt_table, RARRAY_CONST_PTR(labels), VALUE, i+1);
+ MEMCPY(opt_table, RARRAY_CONST_PTR_TRANSIENT(labels), VALUE, i+1);
for (j = 0; j < i+1; j++) {
opt_table[j] &= ~1;
}
@@ -2297,14 +2297,14 @@ iseq_set_exception_table(rb_iseq_t *iseq)
struct iseq_catch_table_entry *entry;
tlen = (int)RARRAY_LEN(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
- tptr = RARRAY_CONST_PTR(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
+ tptr = RARRAY_CONST_PTR_TRANSIENT(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
if (tlen > 0) {
struct iseq_catch_table *table = xmalloc(iseq_catch_table_bytes(tlen));
table->size = tlen;
for (i = 0; i < table->size; i++) {
- ptr = RARRAY_CONST_PTR(tptr[i]);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(tptr[i]);
entry = &table->entries[i];
entry->type = (enum catch_type)(ptr[0] & 0xffff);
entry->start = label_get_position((LABEL *)(ptr[1] & ~1));
diff --git a/debug_counter.h b/debug_counter.h
index 3fcb562..c151e53 100644
--- a/debug_counter.h
+++ b/debug_counter.h
@@ -219,6 +219,11 @@ RB_DEBUG_COUNTER(heap_xmalloc)
RB_DEBUG_COUNTER(heap_xrealloc)
RB_DEBUG_COUNTER(heap_xfree)
+/* transient_heap */
+RB_DEBUG_COUNTER(theap_alloc)
+RB_DEBUG_COUNTER(theap_alloc_fail)
+RB_DEBUG_COUNTER(theap_evacuate)
+
/* load (not implemented yet) */
/*
RB_DEBUG_COUNTER(load_files)
diff --git a/enum.c b/enum.c
index 9485b7e..3be6941 100644
--- a/enum.c
+++ b/enum.c
@@ -14,6 +14,7 @@
#include "ruby/util.h"
#include "id.h"
#include "symbol.h"
+#include "transient_heap.h"
#include <assert.h>
@@ -1171,9 +1172,10 @@ enum_sort_by(VALUE obj)
rb_ary_concat(ary, buf);
}
if (RARRAY_LEN(ary) > 2) {
- RARRAY_PTR_USE(ary, ptr,
- ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE),
- sort_by_cmp, (void *)ary));
+ rb_ary_transient_heap_evacuate(ary, TRUE); /* should be malloc heap */
+ RARRAY_PTR_USE(ary, ptr,
+ ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE),
+ sort_by_cmp, (void *)ary));
}
if (RBASIC(ary)->klass) {
rb_raise(rb_eRuntimeError, "sort_by reentered");
diff --git a/gc.c b/gc.c
index dc8f580..dfccb74 100644
--- a/gc.c
+++ b/gc.c
@@ -35,6 +35,7 @@
#include <sys/types.h>
#include "ruby_assert.h"
#include "debug_counter.h"
+#include "transient_heap.h"
#include "mjit.h"
#undef rb_data_object_wrap
@@ -845,8 +846,6 @@ static void rb_objspace_call_finalizer(rb_objspace_t *objspace);
static VALUE define_final0(VALUE obj, VALUE block);
static void negative_size_allocation_error(const char *);
-static void *aligned_malloc(size_t, size_t);
-static void aligned_free(void *);
static void init_mark_stack(mark_stack_t *stack);
@@ -1190,6 +1189,7 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *pag
{
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
objspace->rgengc.old_objects++;
+ rb_transient_heap_promote(obj);
#if RGENGC_PROFILE >= 2
objspace->profile.total_promoted_count++;
@@ -1486,7 +1486,7 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
objspace->profile.total_freed_pages++;
- aligned_free(GET_PAGE_BODY(page->start));
+ rb_aligned_free(GET_PAGE_BODY(page->start));
free(page);
}
@@ -1524,7 +1524,7 @@ heap_page_allocate(rb_objspace_t *objspace)
int limit = HEAP_PAGE_OBJ_LIMIT;
/* assign heap_page body (contains heap_page_header and RVALUEs) */
- page_body = (struct heap_page_body *)aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
+ page_body = (struct heap_page_body *)rb_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
if (page_body == 0) {
rb_memerror();
}
@@ -1532,7 +1532,7 @@ heap_page_allocate(rb_objspace_t *objspace)
/* assign heap_page entry */
page = (struct heap_page *)calloc(1, sizeof(struct heap_page));
if (page == 0) {
- aligned_free(page_body);
+ rb_aligned_free(page_body);
rb_memerror();
}
@@ -1954,10 +1954,10 @@ newobj_of(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protect
#if GC_DEBUG_STRESS_TO_CLASS
if (UNLIKELY(stress_to_class)) {
- long i, cnt = RARRAY_LEN(stress_to_class);
- for (i = 0; i < cnt; ++i) {
+ long i, cnt = RARRAY_LEN(stress_to_class);
+ for (i = 0; i < cnt; ++i) {
if (klass == RARRAY_AREF(stress_to_class, i)) rb_memerror();
- }
+ }
}
#endif
if (!(during_gc ||
@@ -2259,7 +2259,7 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
rb_str_free(obj);
break;
case T_ARRAY:
- rb_ary_free(obj);
+ rb_ary_free(obj);
break;
case T_HASH:
if (RANY(obj)->as.hash.ntbl) {
@@ -4515,6 +4515,7 @@ static void
gc_mark_ptr(rb_objspace_t *objspace, VALUE obj)
{
if (LIKELY(objspace->mark_func_data == NULL)) {
+ if (RB_TYPE_P(obj, T_NONE)) rb_bug("...");
rgengc_check_relation(objspace, obj);
if (!gc_mark_set(objspace, obj)) return; /* already marked */
gc_aging(objspace, obj);
@@ -4671,16 +4672,24 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
break;
case T_ARRAY:
- if (FL_TEST(obj, ELTS_SHARED)) {
- gc_mark(objspace, any->as.array.as.heap.aux.shared);
+ if (FL_TEST(obj, ELTS_SHARED)) {
+ VALUE root = any->as.array.as.heap.aux.shared;
+ gc_mark(objspace, root);
}
else {
long i, len = RARRAY_LEN(obj);
- const VALUE *ptr = RARRAY_CONST_PTR(obj);
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(obj);
for (i=0; i < len; i++) {
- gc_mark(objspace, *ptr++);
+ gc_mark(objspace, ptr[i]);
}
- }
+
+ if (objspace->mark_func_data == NULL) {
+ if (!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG) &&
+ RARRAY_TRANSIENT_P(obj)) {
+ rb_transient_heap_mark(obj, ptr);
+ }
+ }
+ }
break;
case T_HASH:
@@ -5455,6 +5464,13 @@ rb_gc_verify_internal_consistency(void)
gc_verify_internal_consistency(Qnil);
}
+static VALUE
+gc_verify_transient_heap_internal_consistency(VALUE dmy)
+{
+ rb_transient_heap_verify();
+ return Qnil;
+}
+
/* marks */
static void
@@ -5671,6 +5687,8 @@ gc_marks_finish(rb_objspace_t *objspace)
#endif
}
+ rb_transient_heap_finish_marking();
+
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0);
return TRUE;
@@ -6562,6 +6580,7 @@ gc_start(rb_objspace_t *objspace, int reason)
objspace->profile.heap_used_at_gc_start = heap_allocated_pages;
gc_prof_setup_new_record(objspace, reason);
gc_reset_malloc_info(objspace);
+ rb_transient_heap_start_marking(do_full_mark);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */);
GC_ASSERT(during_gc);
@@ -7812,8 +7831,8 @@ rb_memerror(void)
EC_JUMP_TAG(ec, TAG_RAISE);
}
-static void *
-aligned_malloc(size_t alignment, size_t size)
+void *
+rb_aligned_malloc(size_t alignment, size_t size)
{
void *res;
@@ -7846,8 +7865,8 @@ aligned_malloc(size_t alignment, size_t size)
return res;
}
-static void
-aligned_free(void *ptr)
+void
+rb_aligned_free(void *ptr)
{
#if defined __MINGW32__
__mingw_aligned_free(ptr);
@@ -9551,13 +9570,21 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
#if USE_RGENGC
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
- snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s",
- (void *)obj, age,
- C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
- C(RVALUE_MARK_BITMAP(obj), "M"),
- C(RVALUE_MARKING_BITMAP(obj), "R"),
- C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
- obj_type_name(obj));
+ if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
+ snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s",
+ (void *)obj, age,
+ C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
+ C(RVALUE_MARK_BITMAP(obj), "M"),
+ C(RVALUE_MARKING_BITMAP(obj), "R"),
+ C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
+ obj_type_name(obj));
+ }
+ else {
+ /* fake */
+ snprintf(buff, buff_size, "%p [%dXXXX] %s",
+ (void *)obj, age,
+ obj_type_name(obj));
+ }
#else
snprintf(buff, buff_size, "%p [%s] %s",
(void *)obj,
@@ -9587,10 +9614,25 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
UNEXPECTED_NODE(rb_raw_obj_info);
break;
case T_ARRAY:
- snprintf(buff, buff_size, "%s [%s%s] len: %d", buff,
- C(ARY_EMBED_P(obj), "E"),
- C(ARY_SHARED_P(obj), "S"),
- (int)RARRAY_LEN(obj));
+ if (FL_TEST(obj, ELTS_SHARED)) {
+ snprintf(buff, buff_size, "%s shared -> %s", buff,
+ rb_obj_info(RARRAY(obj)->as.heap.aux.shared));
+ }
+ else if (FL_TEST(obj, RARRAY_EMBED_FLAG)) {
+ snprintf(buff, buff_size, "%s [%s%s] len: %d (embed)", buff,
+ C(ARY_EMBED_P(obj), "E"),
+ C(ARY_SHARED_P(obj), "S"),
+ (int)RARRAY_LEN(obj));
+ }
+ else {
+ snprintf(buff, buff_size, "%s [%s%s%s] len: %d, capa:%d ptr:%p", buff,
+ C(ARY_EMBED_P(obj), "E"),
+ C(ARY_SHARED_P(obj), "S"),
+ C(RARRAY_TRANSIENT_P(obj), "T"),
+ (int)RARRAY_LEN(obj),
+ ARY_EMBED_P(obj) ? -1 : (int)RARRAY(obj)->as.heap.aux.capa,
+ RARRAY_CONST_PTR_TRANSIENT(obj));
+ }
break;
case T_STRING: {
snprintf(buff, buff_size, "%s %s", buff, RSTRING_PTR(obj));
@@ -9954,6 +9996,7 @@ Init_GC(void)
/* internal methods */
rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0);
+ rb_define_singleton_method(rb_mGC, "verify_transient_heap_internal_consistency", gc_verify_transient_heap_internal_consistency, 0);
#if MALLOC_ALLOCATED_SIZE
rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h
index a05651a..1512e78 100644
--- a/include/ruby/ruby.h
+++ b/include/ruby/ruby.h
@@ -1020,12 +1020,15 @@ enum ruby_rarray_flags {
RARRAY_EMBED_LEN_MASK = (RUBY_FL_USER4|RUBY_FL_USER3),
RARRAY_EMBED_LEN_SHIFT = (RUBY_FL_USHIFT+3),
+ RARRAY_TRANSIENT_FLAG = RUBY_FL_USER13,
+
RARRAY_ENUM_END
};
#define RARRAY_EMBED_FLAG (VALUE)RARRAY_EMBED_FLAG
#define RARRAY_EMBED_LEN_MASK (VALUE)RARRAY_EMBED_LEN_MASK
#define RARRAY_EMBED_LEN_MAX RARRAY_EMBED_LEN_MAX
#define RARRAY_EMBED_LEN_SHIFT RARRAY_EMBED_LEN_SHIFT
+#define RARRAY_TRANSIENT_FLAG RARRAY_TRANSIENT_FLAG
struct RArray {
struct RBasic basic;
union {
@@ -1046,9 +1049,14 @@ struct RArray {
#define RARRAY_LEN(a) rb_array_len(a)
#define RARRAY_LENINT(ary) rb_long2int(RARRAY_LEN(ary))
#define RARRAY_CONST_PTR(a) rb_array_const_ptr(a)
+#define RARRAY_CONST_PTR_TRANSIENT(a) rb_array_const_ptr_transient(a)
+#define RARRAY_TRANSIENT_P(ary) FL_TEST_RAW((ary), RARRAY_TRANSIENT_FLAG)
+
+VALUE *rb_ary_ptr_use_start(VALUE ary);
+void rb_ary_ptr_use_end(VALUE ary);
-#define RARRAY_PTR_USE_START(a) ((VALUE *)RARRAY_CONST_PTR(a))
-#define RARRAY_PTR_USE_END(a) /* */
+#define RARRAY_PTR_USE_START(a) rb_ary_ptr_use_start(a)
+#define RARRAY_PTR_USE_END(a) rb_ary_ptr_use_end(a)
#define RARRAY_PTR_USE(ary, ptr_name, expr) do { \
const VALUE _ary = (ary); \
@@ -1057,11 +1065,12 @@ struct RArray {
RARRAY_PTR_USE_END(_ary); \
} while (0)
-#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR(a)[i])
+#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR_TRANSIENT(a)[i])
#define RARRAY_ASET(a, i, v) do { \
const VALUE _ary = (a); \
+ const VALUE _v = (v); \
VALUE *ptr = (VALUE *)RARRAY_PTR_USE_START(_ary); \
- RB_OBJ_WRITE(_ary, &ptr[i], (v)); \
+ RB_OBJ_WRITE(_ary, &ptr[i], _v); \
RARRAY_PTR_USE_END(_ary); \
} while (0)
@@ -2110,12 +2119,23 @@ rb_array_len(VALUE a)
#endif
static inline const VALUE *
-rb_array_const_ptr(VALUE a)
+rb_array_const_ptr_transient(VALUE a)
{
return FIX_CONST_VALUE_PTR((RBASIC(a)->flags & RARRAY_EMBED_FLAG) ?
RARRAY(a)->as.ary : RARRAY(a)->as.heap.ptr);
}
+void rb_ary_detransient(VALUE a);
+
+static inline const VALUE *
+rb_array_const_ptr(VALUE a)
+{
+ if (RARRAY_TRANSIENT_P(a)) {
+ rb_ary_detransient(a);
+ }
+ return rb_array_const_ptr_transient(a);
+}
+
#if defined(EXTLIB) && defined(USE_DLN_A_OUT)
/* hook for external modules */
static char *dln_libs_to_be_linked[] = { EXTLIB, 0 };
diff --git a/inits.c b/inits.c
index c9687de..7eb5431 100644
--- a/inits.c
+++ b/inits.c
@@ -16,6 +16,7 @@
void
rb_call_inits(void)
{
+ CALL(TransientHeap);
CALL(Method);
CALL(RandomSeedCore);
CALL(sym);
diff --git a/insns.def b/insns.def
index 132ce2f..747e6ef 100644
--- a/insns.def
+++ b/insns.def
@@ -524,7 +524,7 @@ newhashfromarray
{
VM_ASSERT(num * 2 == (rb_num_t)RARRAY_LEN(ary));
hash = rb_hash_new_with_size(num);
- rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR(ary), hash);
+ rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR_TRANSIENT(ary), hash);
}
/* put new Range object.(Range.new(low, high, flag)) */
diff --git a/internal.h b/internal.h
index 697a119..ffe8452 100644
--- a/internal.h
+++ b/internal.h
@@ -1073,6 +1073,26 @@ VALUE rb_gvar_set(struct rb_global_entry *, VALUE);
VALUE rb_gvar_defined(struct rb_global_entry *);
/* array.c */
+
+#ifndef ARRAY_DEBUG
+#define ARRAY_DEBUG 0
+#endif
+
+#ifdef ARRAY_DEBUG
+#define RARRAY_PTR_IN_USE_FLAG FL_USER14
+#define ARY_PTR_USING_P(ary) FL_TEST_RAW((ary), RARRAY_PTR_IN_USE_FLAG)
+
+#else
+
+/* disable debug function */
+#undef RARRAY_PTR_USE_START
+#undef RARRAY_PTR_USE_END
+#define RARRAY_PTR_USE_START(a) ((VALUE *)RARRAY_CONST_PTR_TRANSIENT(a))
+#define RARRAY_PTR_USE_END(a)
+#define ARY_PTR_USING_P(ary) 0
+
+#endif
+
VALUE rb_ary_last(int, const VALUE *, VALUE);
void rb_ary_set_len(VALUE, long);
void rb_ary_delete_same(VALUE, VALUE);
@@ -1100,7 +1120,7 @@ static inline VALUE
rb_ary_entry_internal(VALUE ary, long offset)
{
long len = RARRAY_LEN(ary);
- const VALUE *ptr = RARRAY_CONST_PTR(ary);
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
if (len == 0) return Qnil;
if (offset < 0) {
offset += len;
@@ -1337,6 +1357,9 @@ RUBY_SYMBOL_EXPORT_END
rb_wb_unprotected_newobj_of(klass, flags))
#define NEWOBJ_OF(obj,type,klass,flags) RB_NEWOBJ_OF(obj,type,klass,flags)
+void *rb_aligned_malloc(size_t, size_t);
+void rb_aligned_free(void *);
+
/* hash.c */
struct st_table *rb_hash_tbl_raw(VALUE hash);
VALUE rb_hash_new_with_size(st_index_t size);
diff --git a/string.c b/string.c
index e654a30..48718f9 100644
--- a/string.c
+++ b/string.c
@@ -2006,9 +2006,9 @@ rb_str_format_m(VALUE str, VALUE arg)
VALUE tmp = rb_check_array_type(arg);
if (!NIL_P(tmp)) {
- VALUE rv = rb_str_format(RARRAY_LENINT(tmp), RARRAY_CONST_PTR(tmp), str);
- RB_GC_GUARD(tmp);
- return rv;
+ const long len = RARRAY_LENINT(tmp);
+ VALUE rv = rb_str_format(len, RARRAY_CONST_PTR(tmp), str);
+ return rv;
}
return rb_str_format(1, &arg, str);
}
diff --git a/test/ruby/test_enum.rb b/test/ruby/test_enum.rb
index a4eace2..c56e280e 100644
--- a/test/ruby/test_enum.rb
+++ b/test/ruby/test_enum.rb
@@ -1115,4 +1115,19 @@ class TestEnumerable < Test::Unit::TestCase
assert_equal([1, 2, 3, 4, 5, 10], (1..100).uniq{|x| (x**2) % 10 }.first(6))
assert_equal([1, [1, 2]], Foo.new.to_enum.uniq)
end
+
+ def test_transient_heap_sort_by
+ klass = Class.new do
+ include Comparable
+ attr_reader :i
+ def initialize e
+ @i = e
+ end
+ def <=> other
+ GC.start
+ i <=> other.i
+ end
+ end
+ assert_equal [1, 2, 3, 4, 5], (1..5).sort_by{|e| klass.new e}
+ end
end
diff --git a/vm_args.c b/vm_args.c
index 4e98957..884023b 100644
--- a/vm_args.c
+++ b/vm_args.c
@@ -164,7 +164,7 @@ args_copy(struct args_info *args)
static inline const VALUE *
args_rest_argv(struct args_info *args)
{
- return RARRAY_CONST_PTR(args->rest) + args->rest_index;
+ return RARRAY_CONST_PTR_TRANSIENT(args->rest) + args->rest_index;
}
static inline VALUE
@@ -314,7 +314,7 @@ args_setup_post_parameters(struct args_info *args, int argc, VALUE *locals)
{
long len;
len = RARRAY_LEN(args->rest);
- MEMCPY(locals, RARRAY_CONST_PTR(args->rest) + len - argc, VALUE, argc);
+ MEMCPY(locals, RARRAY_CONST_PTR_TRANSIENT(args->rest) + len - argc, VALUE, argc);
rb_ary_resize(args->rest, len - argc);
}
@@ -335,7 +335,7 @@ args_setup_opt_parameters(struct args_info *args, int opt_max, VALUE *locals)
if (args->rest) {
int len = RARRAY_LENINT(args->rest);
- const VALUE *argv = RARRAY_CONST_PTR(args->rest);
+ const VALUE *argv = RARRAY_CONST_PTR_TRANSIENT(args->rest);
for (; i<opt_max && args->rest_index < len; i++, args->rest_index++) {
locals[i] = argv[args->rest_index];
@@ -785,7 +785,7 @@ vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calli
cfp->sp--;
if (!NIL_P(ary)) {
- const VALUE *ptr = RARRAY_CONST_PTR(ary);
+ const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
long len = RARRAY_LEN(ary), i;
CHECK_VM_STACK_OVERFLOW(cfp, len);
diff --git a/vm_eval.c b/vm_eval.c
index 1176c49..a864b75 100644
--- a/vm_eval.c
+++ b/vm_eval.c
@@ -764,7 +764,7 @@ rb_apply(VALUE recv, ID mid, VALUE args)
return ret;
}
argv = ALLOCA_N(VALUE, argc);
- MEMCPY(argv, RARRAY_CONST_PTR(args), VALUE, argc);
+ MEMCPY(argv, RARRAY_CONST_PTR_TRANSIENT(args), VALUE, argc);
return rb_call(recv, mid, argc, argv, CALL_FCALL);
}
diff --git a/vm_insnhelper.c b/vm_insnhelper.c
index e162e9d..5904ce7 100644
--- a/vm_insnhelper.c
+++ b/vm_insnhelper.c
@@ -1265,7 +1265,7 @@ vm_expandarray(VALUE *sp, VALUE ary, rb_num_t num, int flag)
len = 1;
}
else {
- ptr = RARRAY_CONST_PTR(ary);
+ ptr = RARRAY_CONST_PTR_TRANSIENT(ary);
len = (rb_num_t)RARRAY_LEN(ary);
}