summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2013-11-05 04:51:01 +0000
committerko1 <ko1@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2013-11-05 04:51:01 +0000
commitd1674ef67b61df4ff27e93772c0946ae74869c5d (patch)
tree8ce4da84dab8b716b48695000f062fae92b201f1
parent5ba523967ec02f9abed15a219f776c3242e0e742 (diff)
* gc.c: add support to estimate increase of oldspace memory usage.
This is another approach to solve an issue discussed at r43530. This feature is diabled as default. This feature measures an increment of memory consuption by oldgen objects. It measures memory consumption for each objects when the object is promoted. However, measurement of memory consumption is not accurate now. So that this measurement is `estimation'. To implement this feature, move memsize_of() function from ext/objspace/objspace.c and expose rb_obj_memsize_of(). Some memsize() functions for T_DATA (T_TYPEDDATA) have problem to measure memory size, so that we ignores T_DATA objects now. For example, some functions skip NULL check for pointer. The macro RGENGC_ESTIMATE_OLDSPACE enables/disables this feature, and turned off as default. We need to compare 3gen GC and this feature carefully. (it is possible to enable both feature) We need a help to compare them. * internal.h: expose rb_obj_memsize_of(). * ext/objspace/objspace.c: use rb_obj_memsize_of() function. * cont.c (fiber_memsize): fix to check NULL. * variable.c (autoload_memsize): ditto. * vm.c (vm_memsize): ditto. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@43532 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r--ChangeLog35
-rw-r--r--cont.c3
-rw-r--r--ext/objspace/objspace.c132
-rw-r--r--gc.c200
-rw-r--r--internal.h1
-rw-r--r--variable.c2
-rw-r--r--vm.c4
7 files changed, 247 insertions, 130 deletions
diff --git a/ChangeLog b/ChangeLog
index 0ba325ec68b..d1f07e511f6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,38 @@
+Tue Nov 5 13:37:19 2013 Koichi Sasada <ko1@atdot.net>
+
+ * gc.c: add support to estimate increase of oldspace memory usage.
+ This is another approach to solve an issue discussed at r43530.
+ This feature is diabled as default.
+
+ This feature measures an increment of memory consuption by oldgen
+ objects. It measures memory consumption for each objects when
+ the object is promoted. However, measurement of memory consumption
+ is not accurate now. So that this measurement is `estimation'.
+
+ To implement this feature, move memsize_of() function from
+ ext/objspace/objspace.c and expose rb_obj_memsize_of().
+
+ Some memsize() functions for T_DATA (T_TYPEDDATA) have problem to
+ measure memory size, so that we ignores T_DATA objects now.
+ For example, some functions skip NULL check for pointer.
+
+ The macro RGENGC_ESTIMATE_OLDSPACE enables/disables this feature,
+ and turned off as default.
+
+ We need to compare 3gen GC and this feature carefully.
+ (it is possible to enable both feature)
+ We need a help to compare them.
+
+ * internal.h: expose rb_obj_memsize_of().
+
+ * ext/objspace/objspace.c: use rb_obj_memsize_of() function.
+
+ * cont.c (fiber_memsize): fix to check NULL.
+
+ * variable.c (autoload_memsize): ditto.
+
+ * vm.c (vm_memsize): ditto.
+
Tue Nov 5 04:03:07 2013 Koichi Sasada <ko1@atdot.net>
* gc.c (GC_MALLOC_LIMIT_MAX): fix default value 512MB -> 384MB.
diff --git a/cont.c b/cont.c
index b9760569550..6ac7a1ea57f 100644
--- a/cont.c
+++ b/cont.c
@@ -351,7 +351,8 @@ fiber_memsize(const void *ptr)
size_t size = 0;
if (ptr) {
size = sizeof(*fib);
- if (fib->cont.type != ROOT_FIBER_CONTEXT) {
+ if (fib->cont.type != ROOT_FIBER_CONTEXT &&
+ fib->cont.saved_thread.local_storage != NULL) {
size += st_memsize(fib->cont.saved_thread.local_storage);
}
size += cont_memsize(&fib->cont);
diff --git a/ext/objspace/objspace.c b/ext/objspace/objspace.c
index b0df466ee36..82a743ba5cb 100644
--- a/ext/objspace/objspace.c
+++ b/ext/objspace/objspace.c
@@ -18,131 +18,9 @@
#include <ruby/re.h>
#include "node.h"
#include "gc.h"
-#include "regint.h"
#include "internal.h"
-size_t rb_str_memsize(VALUE);
-size_t rb_ary_memsize(VALUE);
-size_t rb_io_memsize(const rb_io_t *);
-size_t rb_generic_ivar_memsize(VALUE);
-size_t rb_objspace_data_type_memsize(VALUE obj);
-
-static size_t
-memsize_of(VALUE obj)
-{
- size_t size = 0;
-
- if (SPECIAL_CONST_P(obj)) {
- return 0;
- }
-
- if (FL_TEST(obj, FL_EXIVAR)) {
- size += rb_generic_ivar_memsize(obj);
- }
-
- switch (BUILTIN_TYPE(obj)) {
- case T_OBJECT:
- if (!(RBASIC(obj)->flags & ROBJECT_EMBED) &&
- ROBJECT(obj)->as.heap.ivptr) {
- size += ROBJECT(obj)->as.heap.numiv * sizeof(VALUE);
- }
- break;
- case T_MODULE:
- case T_CLASS:
- if (RCLASS_M_TBL(obj)) {
- size += st_memsize(RCLASS_M_TBL(obj));
- }
- if (RCLASS_IV_TBL(obj)) {
- size += st_memsize(RCLASS_IV_TBL(obj));
- }
- if (RCLASS_IV_INDEX_TBL(obj)) {
- size += st_memsize(RCLASS_IV_INDEX_TBL(obj));
- }
- if (RCLASS(obj)->ptr->iv_tbl) {
- size += st_memsize(RCLASS(obj)->ptr->iv_tbl);
- }
- if (RCLASS(obj)->ptr->const_tbl) {
- size += st_memsize(RCLASS(obj)->ptr->const_tbl);
- }
- size += sizeof(rb_classext_t);
- break;
- case T_STRING:
- size += rb_str_memsize(obj);
- break;
- case T_ARRAY:
- size += rb_ary_memsize(obj);
- break;
- case T_HASH:
- if (RHASH(obj)->ntbl) {
- size += st_memsize(RHASH(obj)->ntbl);
- }
- break;
- case T_REGEXP:
- if (RREGEXP(obj)->ptr) {
- size += onig_memsize(RREGEXP(obj)->ptr);
- }
- break;
- case T_DATA:
- size += rb_objspace_data_type_memsize(obj);
- break;
- case T_MATCH:
- if (RMATCH(obj)->rmatch) {
- struct rmatch *rm = RMATCH(obj)->rmatch;
- size += onig_region_memsize(&rm->regs);
- size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
- size += sizeof(struct rmatch);
- }
- break;
- case T_FILE:
- if (RFILE(obj)->fptr) {
- size += rb_io_memsize(RFILE(obj)->fptr);
- }
- break;
- case T_RATIONAL:
- case T_COMPLEX:
- break;
- case T_ICLASS:
- /* iClass shares table with the module */
- break;
-
- case T_FLOAT:
- break;
-
- case T_BIGNUM:
- if (!(RBASIC(obj)->flags & RBIGNUM_EMBED_FLAG) && RBIGNUM_DIGITS(obj)) {
- size += RBIGNUM_LEN(obj) * sizeof(BDIGIT);
- }
- break;
- case T_NODE:
- switch (nd_type(obj)) {
- case NODE_SCOPE:
- if (RNODE(obj)->u1.tbl) {
- /* TODO: xfree(RANY(obj)->as.node.u1.tbl); */
- }
- break;
- case NODE_ALLOCA:
- /* TODO: xfree(RANY(obj)->as.node.u1.node); */
- ;
- }
- break; /* no need to free iv_tbl */
-
- case T_STRUCT:
- if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
- RSTRUCT(obj)->as.heap.ptr) {
- size += sizeof(VALUE) * RSTRUCT_LEN(obj);
- }
- break;
-
- case T_ZOMBIE:
- break;
-
- default:
- rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
- BUILTIN_TYPE(obj), (void*)obj);
- }
-
- return size;
-}
+size_t rb_obj_memsize_of(VALUE);
/*
* call-seq:
@@ -160,7 +38,7 @@ memsize_of(VALUE obj)
static VALUE
memsize_of_m(VALUE self, VALUE obj)
{
- return SIZET2NUM(memsize_of(obj));
+ return SIZET2NUM(rb_obj_memsize_of(obj));
}
struct total_data {
@@ -187,7 +65,7 @@ total_i(void *vstart, void *vend, size_t stride, void *ptr)
continue;
default:
if (data->klass == 0 || rb_obj_is_kind_of(v, data->klass)) {
- data->total += memsize_of(v);
+ data->total += rb_obj_memsize_of(v);
}
}
}
@@ -254,7 +132,7 @@ cos_i(void *vstart, void *vend, size_t stride, void *data)
for (;v != (VALUE)vend; v += stride) {
if (RBASIC(v)->flags) {
- counts[BUILTIN_TYPE(v)] += memsize_of(v);
+ counts[BUILTIN_TYPE(v)] += rb_obj_memsize_of(v);
}
}
return 0;
@@ -637,7 +515,7 @@ static size_t
iow_size(const void *ptr)
{
VALUE obj = (VALUE)ptr;
- return memsize_of(obj);
+ return rb_obj_memsize_of(obj);
}
static const rb_data_type_t iow_data_type = {
diff --git a/gc.c b/gc.c
index a5671f570cb..4984f890d6d 100644
--- a/gc.c
+++ b/gc.c
@@ -89,6 +89,7 @@ rb_gc_guarded_ptr(volatile VALUE *ptr)
#ifndef GC_HEAP_GROWTH_MAX
#define GC_HEAP_GROWTH_MAX 0 /* 0 is disable */
#endif
+
#ifndef GC_MALLOC_LIMIT
#define GC_MALLOC_LIMIT (16 /* 16 MB */ * 1024 * 1024 /* 1MB */)
#endif
@@ -99,6 +100,16 @@ rb_gc_guarded_ptr(volatile VALUE *ptr)
#define GC_MALLOC_LIMIT_GROWTH_FACTOR 2.0
#endif
+#ifndef GC_HEAP_OLDSPACE_MIN
+#define GC_HEAP_OLDSPACE_MIN (16 /* 16 MB */ * 1024 * 1024 /* 1MB */)
+#endif
+#ifndef GC_HEAP_OLDSPACE_GROWTH_FACTOR
+#define GC_HEAP_OLDSPACE_GROWTH_FACTOR 1.8
+#endif
+#ifndef GC_HEAP_OLDSPACE_MAX
+#define GC_HEAP_OLDSPACE_MAX (384 /* 384 MB */ * 1024 * 1024 /* 1MB */)
+#endif
+
typedef struct {
unsigned int initial_heap_min_slots;
unsigned int initial_heap_min_free_slots;
@@ -172,6 +183,16 @@ static ruby_gc_params_t initial_params = {
#define RGENGC_THREEGEN 0
#endif
+/* RGENGC_ESTIMATE_OLDSPACE
+ * Enable/disable to estimate increase size of oldspace.
+ * If estimation exceeds threashold, then will invoke full GC.
+ * 0: disable estimation.
+ * 1: enable estimation.
+ */
+#ifndef RGENGC_ESTIMATE_OLDSPACE
+#define RGENGC_ESTIMATE_OLDSPACE 0
+#endif
+
#else /* USE_RGENGC */
#define RGENGC_DEBUG 0
#define RGENGC_CHECK_MODE 0
@@ -455,6 +476,11 @@ typedef struct rb_objspace {
size_t young_object_count;
#endif
+#if RGENGC_ESTIMATE_OLDSPACE
+ size_t oldspace_increase;
+ size_t oldspace_increase_limit;
+#endif
+
#if RGENGC_CHECK_MODE >= 2
/* for check mode */
VALUE parent_object;
@@ -612,6 +638,8 @@ static void gc_mark(rb_objspace_t *objspace, VALUE ptr);
static void gc_mark_maybe(rb_objspace_t *objspace, VALUE ptr);
static void gc_mark_children(rb_objspace_t *objspace, VALUE ptr);
+static size_t obj_memsize_of(VALUE obj, int use_tdata);
+
static double getrusage_time(void);
static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, int reason);
static inline void gc_prof_timer_start(rb_objspace_t *);
@@ -656,6 +684,8 @@ check_gen_consistency(VALUE obj)
int promoted_flag = FL_TEST2(obj, FL_PROMOTED);
rb_objspace_t *objspace = &rb_objspace;
+ obj_memsize_of((VALUE)obj, FALSE);
+
if (!is_pointer_to_heap(objspace, (void *)obj)) {
rb_bug("check_gen_consistency: %p (%s) is not Ruby object.", (void *)obj, obj_type_name(obj));
}
@@ -816,6 +846,10 @@ rb_objspace_alloc(void)
malloc_limit = initial_malloc_limit;
ruby_gc_stress = ruby_initial_gc_stress;
+#if RGENGC_ESTIMATE_OLDSPACE
+ objspace->rgengc.oldspace_increase_limit = GC_HEAP_OLDSPACE_MIN;
+#endif
+
return objspace;
}
#endif
@@ -2316,6 +2350,137 @@ rb_obj_id(VALUE obj)
return nonspecial_obj_id(obj);
}
+size_t rb_str_memsize(VALUE);
+size_t rb_ary_memsize(VALUE);
+size_t rb_io_memsize(const rb_io_t *);
+size_t rb_generic_ivar_memsize(VALUE);
+#include "regint.h"
+
+static size_t
+obj_memsize_of(VALUE obj, int use_tdata)
+{
+ size_t size = 0;
+
+ if (SPECIAL_CONST_P(obj)) {
+ return 0;
+ }
+
+ if (FL_TEST(obj, FL_EXIVAR)) {
+ size += rb_generic_ivar_memsize(obj);
+ }
+
+ switch (BUILTIN_TYPE(obj)) {
+ case T_OBJECT:
+ if (!(RBASIC(obj)->flags & ROBJECT_EMBED) &&
+ ROBJECT(obj)->as.heap.ivptr) {
+ size += ROBJECT(obj)->as.heap.numiv * sizeof(VALUE);
+ }
+ break;
+ case T_MODULE:
+ case T_CLASS:
+ if (RCLASS_M_TBL(obj)) {
+ size += st_memsize(RCLASS_M_TBL(obj));
+ }
+ if (RCLASS_EXT(obj)) {
+ if (RCLASS_IV_TBL(obj)) {
+ size += st_memsize(RCLASS_IV_TBL(obj));
+ }
+ if (RCLASS_IV_INDEX_TBL(obj)) {
+ size += st_memsize(RCLASS_IV_INDEX_TBL(obj));
+ }
+ if (RCLASS(obj)->ptr->iv_tbl) {
+ size += st_memsize(RCLASS(obj)->ptr->iv_tbl);
+ }
+ if (RCLASS(obj)->ptr->const_tbl) {
+ size += st_memsize(RCLASS(obj)->ptr->const_tbl);
+ }
+ size += sizeof(rb_classext_t);
+ }
+ break;
+ case T_STRING:
+ size += rb_str_memsize(obj);
+ break;
+ case T_ARRAY:
+ size += rb_ary_memsize(obj);
+ break;
+ case T_HASH:
+ if (RHASH(obj)->ntbl) {
+ size += st_memsize(RHASH(obj)->ntbl);
+ }
+ break;
+ case T_REGEXP:
+ if (RREGEXP(obj)->ptr) {
+ size += onig_memsize(RREGEXP(obj)->ptr);
+ }
+ break;
+ case T_DATA:
+ if (use_tdata) size += rb_objspace_data_type_memsize(obj);
+ break;
+ case T_MATCH:
+ if (RMATCH(obj)->rmatch) {
+ struct rmatch *rm = RMATCH(obj)->rmatch;
+ size += onig_region_memsize(&rm->regs);
+ size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
+ size += sizeof(struct rmatch);
+ }
+ break;
+ case T_FILE:
+ if (RFILE(obj)->fptr) {
+ size += rb_io_memsize(RFILE(obj)->fptr);
+ }
+ break;
+ case T_RATIONAL:
+ case T_COMPLEX:
+ break;
+ case T_ICLASS:
+ /* iClass shares table with the module */
+ break;
+
+ case T_FLOAT:
+ break;
+
+ case T_BIGNUM:
+ if (!(RBASIC(obj)->flags & RBIGNUM_EMBED_FLAG) && RBIGNUM_DIGITS(obj)) {
+ size += RBIGNUM_LEN(obj) * sizeof(BDIGIT);
+ }
+ break;
+ case T_NODE:
+ switch (nd_type(obj)) {
+ case NODE_SCOPE:
+ if (RNODE(obj)->u1.tbl) {
+ /* TODO: xfree(RANY(obj)->as.node.u1.tbl); */
+ }
+ break;
+ case NODE_ALLOCA:
+ /* TODO: xfree(RANY(obj)->as.node.u1.node); */
+ ;
+ }
+ break; /* no need to free iv_tbl */
+
+ case T_STRUCT:
+ if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
+ RSTRUCT(obj)->as.heap.ptr) {
+ size += sizeof(VALUE) * RSTRUCT_LEN(obj);
+ }
+ break;
+
+ case T_ZOMBIE:
+ break;
+
+ default:
+ rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
+ BUILTIN_TYPE(obj), (void*)obj);
+ }
+
+ return size;
+}
+
+size_t
+rb_obj_memsize_of(VALUE obj)
+{
+ return obj_memsize_of(obj, TRUE);
+}
+
static int
set_zero(st_data_t key, st_data_t val, st_data_t arg)
{
@@ -2691,6 +2856,27 @@ gc_after_sweep(rb_objspace_t *objspace)
#endif
}
+#if RGENGC_ESTIMATE_OLDSPACE
+ if (objspace->rgengc.oldspace_increase > objspace->rgengc.oldspace_increase_limit) {
+ objspace->rgengc.need_major_gc = TRUE;
+ objspace->rgengc.oldspace_increase_limit *= GC_HEAP_OLDSPACE_GROWTH_FACTOR;
+ objspace->rgengc.oldspace_increase = 0;
+ if (objspace->rgengc.oldspace_increase_limit > GC_HEAP_OLDSPACE_MAX) {
+ objspace->rgengc.oldspace_increase_limit = GC_HEAP_OLDSPACE_MAX;
+ }
+ }
+ else {
+ objspace->rgengc.oldspace_increase_limit /= ((GC_HEAP_OLDSPACE_GROWTH_FACTOR - 1)/10 + 1);
+ if (objspace->rgengc.oldspace_increase_limit < GC_HEAP_OLDSPACE_MIN) {
+ objspace->rgengc.oldspace_increase_limit = GC_HEAP_OLDSPACE_MIN;
+ }
+ }
+ if (0) fprintf(stderr, "%d\t%d\t%u\t%u\t%d\n", (int)rb_gc_count(), objspace->rgengc.need_major_gc,
+ (unsigned int)objspace->rgengc.oldspace_increase,
+ (unsigned int)objspace->rgengc.oldspace_increase_limit,
+ (unsigned int)GC_HEAP_OLDSPACE_MAX);
+#endif
+
gc_prof_set_heap_info(objspace);
heap_pages_free_unused_pages(objspace);
@@ -3420,6 +3606,11 @@ gc_mark_children(rb_objspace_t *objspace, VALUE ptr)
/* infant -> old */
objspace->rgengc.old_object_count++;
objspace->rgengc.parent_object_is_old = TRUE;
+
+#if RGENGC_ESTIMATE_OLDSPACE
+ objspace->rgengc.oldspace_increase += obj_memsize_of((VALUE)obj, FALSE);
+#endif
+
#endif
rgengc_report(3, objspace, "gc_mark_children: promote infant -> young %p (%s).\n", (void *)obj, obj_type_name((VALUE)obj));
}
@@ -3431,6 +3622,9 @@ gc_mark_children(rb_objspace_t *objspace, VALUE ptr)
/* young -> old */
RVALUE_PROMOTE_YOUNG((VALUE)obj);
objspace->rgengc.old_object_count++;
+#if RGENGC_ESTIMATE_OLDSPACE
+ objspace->rgengc.oldspace_increase += obj_memsize_of((VALUE)obj, FALSE);
+#endif
rgengc_report(3, objspace, "gc_mark_children: promote young -> old %p (%s).\n", (void *)obj, obj_type_name((VALUE)obj));
}
else {
@@ -5044,6 +5238,12 @@ rb_gc_set_params(void)
get_envparam_int ("RUBY_GC_MALLOC_LIMIT", &initial_malloc_limit, 0);
get_envparam_int ("RUBY_GC_MALLOC_LIMIT_MAX", &initial_malloc_limit_max, 0);
get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &initial_malloc_limit_growth_factor, 1.0);
+
+ /*
+ get_envparam_int("RUBY_GC_HEAP_OLDSPACE_MIN", &initial_oldspace_min, 0);
+ get_envparam_int("RUBY_GC_HEAP_OLDSPACE_MAX", &initial_oldspace_max, 0);
+ get_envparam_int("RUBY_GC_HEAP_OLDSPACE_GROWTH_FACTOR", &initial_oldspace_growth_factor, 0);
+ */
}
void
diff --git a/internal.h b/internal.h
index 77e2bd090b4..20a813ca798 100644
--- a/internal.h
+++ b/internal.h
@@ -819,6 +819,7 @@ st_table *rb_st_copy(VALUE obj, struct st_table *orig_tbl);
/* gc.c */
size_t rb_gc_count();
+size_t rb_obj_memsize_of(VALUE);
RUBY_SYMBOL_EXPORT_END
diff --git a/variable.c b/variable.c
index 1c7bfb030f3..a1bb36e0880 100644
--- a/variable.c
+++ b/variable.c
@@ -1529,7 +1529,7 @@ static size_t
autoload_memsize(const void *ptr)
{
const st_table *tbl = ptr;
- return st_memsize(tbl);
+ return tbl ? st_memsize(tbl) : 0;
}
static const rb_data_type_t autoload_data_type = {
diff --git a/vm.c b/vm.c
index 830e66cf344..bf965b2426b 100644
--- a/vm.c
+++ b/vm.c
@@ -1673,7 +1673,9 @@ vm_memsize(const void *ptr)
if (ptr) {
const rb_vm_t *vmobj = ptr;
size_t size = sizeof(rb_vm_t);
- size += st_memsize(vmobj->living_threads);
+ if (vmobj->living_threads) {
+ size += st_memsize(vmobj->living_threads);
+ }
if (vmobj->defined_strings) {
size += DEFINED_EXPR * sizeof(VALUE);
}