diff options
Diffstat (limited to 'imemo.c')
| -rw-r--r-- | imemo.c | 325 |
1 files changed, 226 insertions, 99 deletions
@@ -29,7 +29,10 @@ rb_imemo_name(enum imemo_type type) IMEMO_NAME(svar); IMEMO_NAME(throw_data); IMEMO_NAME(tmpbuf); + IMEMO_NAME(cvar_entry); IMEMO_NAME(fields); + IMEMO_NAME(subclasses); + IMEMO_NAME(cdhash); #undef IMEMO_NAME } rb_bug("unreachable"); @@ -40,110 +43,125 @@ rb_imemo_name(enum imemo_type type) * ========================================================================= */ VALUE -rb_imemo_new(enum imemo_type type, VALUE v0, size_t size) +rb_imemo_new(enum imemo_type type, VALUE v0, size_t size, bool is_shareable) { - VALUE flags = T_IMEMO | FL_WB_PROTECTED | (type << FL_USHIFT); - NEWOBJ_OF(obj, void, v0, flags, size, 0); - - return (VALUE)obj; + VALUE flags = T_IMEMO | (type << FL_USHIFT) | (is_shareable ? FL_SHAREABLE : 0); + return rb_newobj_of(v0, flags, size); } -static rb_imemo_tmpbuf_t * +VALUE rb_imemo_tmpbuf_new(void) { - size_t size = sizeof(struct rb_imemo_tmpbuf_struct); VALUE flags = T_IMEMO | (imemo_tmpbuf << FL_USHIFT); - NEWOBJ_OF(obj, struct rb_imemo_tmpbuf_struct, 0, flags, size, 0); + UNPROTECTED_NEWOBJ_OF(obj, rb_imemo_tmpbuf_t, 0, flags, sizeof(rb_imemo_tmpbuf_t)); + + rb_gc_register_pinning_obj((VALUE)obj); + + obj->ptr = NULL; + obj->size = 0; - return obj; + return (VALUE)obj; } void * -rb_alloc_tmp_buffer_with_count(volatile VALUE *store, size_t size, size_t cnt) +rb_alloc_tmp_buffer(volatile VALUE *store, long len) { - void *ptr; - rb_imemo_tmpbuf_t *tmpbuf; + if (len < 0) { + rb_raise(rb_eArgError, "negative buffer size (or size too big)"); + } /* Keep the order; allocate an empty imemo first then xmalloc, to * get rid of potential memory leak */ - tmpbuf = rb_imemo_tmpbuf_new(); + rb_imemo_tmpbuf_t *tmpbuf = (rb_imemo_tmpbuf_t *)rb_imemo_tmpbuf_new(); *store = (VALUE)tmpbuf; - ptr = ruby_xmalloc(size); + void *ptr = ruby_xmalloc(len); tmpbuf->ptr = ptr; - tmpbuf->cnt = cnt; + tmpbuf->size = len; return ptr; } void * -rb_alloc_tmp_buffer(volatile VALUE *store, long len) +rb_alloc_tmp_buffer_with_count(volatile VALUE *store, size_t size, size_t cnt) { - long cnt; - - if (len < 0 || (cnt = (long)roomof(len, sizeof(VALUE))) < 0) { - rb_raise(rb_eArgError, "negative buffer size (or size too big)"); - } - - return rb_alloc_tmp_buffer_with_count(store, len, cnt); + return rb_alloc_tmp_buffer(store, (long)size); } void rb_free_tmp_buffer(volatile VALUE *store) { + if (!*store) return; rb_imemo_tmpbuf_t *s = (rb_imemo_tmpbuf_t*)ATOMIC_VALUE_EXCHANGE(*store, 0); if (s) { void *ptr = ATOMIC_PTR_EXCHANGE(s->ptr, 0); - s->cnt = 0; - ruby_xfree(ptr); + long size = s->size; + s->size = 0; + ruby_xfree_sized(ptr, size); } } -rb_imemo_tmpbuf_t * -rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt) +struct MEMO * +rb_imemo_memo_new(VALUE a, VALUE b, long c) { - rb_imemo_tmpbuf_t *tmpbuf = rb_imemo_tmpbuf_new(); - tmpbuf->ptr = buf; - tmpbuf->next = old_heap; - tmpbuf->cnt = cnt; + struct MEMO *memo = IMEMO_NEW(struct MEMO, imemo_memo, 0); + + *((VALUE *)&memo->v1) = a; + *((VALUE *)&memo->v2) = b; + memo->u3.cnt = c; - return tmpbuf; + return memo; } -static VALUE -imemo_fields_new(VALUE owner, size_t capa) +struct MEMO * +rb_imemo_memo_new_value(VALUE a, VALUE b, VALUE c) { - size_t embedded_size = offsetof(struct rb_fields, as.embed) + capa * sizeof(VALUE); - if (rb_gc_size_allocatable_p(embedded_size)) { - VALUE fields = rb_imemo_new(imemo_fields, owner, embedded_size); - RUBY_ASSERT(IMEMO_TYPE_P(fields, imemo_fields)); - return fields; - } - else { - VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields)); - FL_SET_RAW(fields, OBJ_FIELD_EXTERNAL); - IMEMO_OBJ_FIELDS(fields)->as.external.ptr = ALLOC_N(VALUE, capa); - return fields; - } + struct MEMO *memo = IMEMO_NEW(struct MEMO, imemo_memo, 0); + + *((VALUE *)&memo->v1) = a; + *((VALUE *)&memo->v2) = b; + *((VALUE *)&memo->u3.value) = c; + memo->flags |= MEMO_U3_IS_VALUE; + + return memo; } VALUE -rb_imemo_fields_new(VALUE owner, size_t capa) +rb_imemo_cdhash_new(size_t size, const struct st_hash_type *type) { - return imemo_fields_new(owner, capa); + struct rb_imemo_cdhash *memo = IMEMO_NEW(struct rb_imemo_cdhash, imemo_cdhash, 0); + memo->tbl.num_entries = 0; + st_init_existing_table_with_size(&memo->tbl, type, size); + return (VALUE)memo; } -static VALUE -imemo_fields_new_complex(VALUE owner, size_t capa) +VALUE +rb_imemo_fields_new(VALUE owner, shape_id_t shape_id, bool shareable) { - VALUE fields = imemo_fields_new(owner, sizeof(struct rb_fields)); - IMEMO_OBJ_FIELDS(fields)->as.complex.table = st_init_numtable_with_size(capa); + size_t capa = RSHAPE_CAPACITY(shape_id); + size_t embedded_size = offsetof(struct rb_fields, as.embed) + capa * sizeof(VALUE); + RUBY_ASSERT(rb_gc_size_allocatable_p(embedded_size)); + VALUE fields = rb_imemo_new(imemo_fields, owner, embedded_size, shareable); + // imemo fields objects should always have "RObject" layout. The + // layout in the shape describes the layout of the thing on which it is set. + // Imemo fields have the same layout as robject, therefore the layout + // should reflect that fact. + RBASIC_SET_SHAPE_ID(fields, rb_shape_id_with_robject_layout(shape_id)); + RUBY_ASSERT(IMEMO_TYPE_P(fields, imemo_fields)); return fields; } VALUE -rb_imemo_fields_new_complex(VALUE owner, size_t capa) +rb_imemo_fields_new_complex(VALUE owner, shape_id_t shape_id, size_t capa, bool shareable) { - return imemo_fields_new_complex(owner, capa); + VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable); + IMEMO_OBJ_FIELDS(fields)->as.complex.table = st_init_numtable_with_size(capa); + FL_SET_RAW(fields, OBJ_FIELD_HEAP); + // imemo fields objects should always have "RObject" layout. The + // layout in the shape describes the layout of the thing on which it is set. + // Imemo fields have the same layout as robject, therefore the layout + // should reflect that fact. + RBASIC_SET_SHAPE_ID(fields, rb_shape_id_with_robject_layout(shape_id)); + return fields; } static int @@ -162,10 +180,16 @@ imemo_fields_complex_wb_i(st_data_t key, st_data_t value, st_data_t arg) } VALUE -rb_imemo_fields_new_complex_tbl(VALUE owner, st_table *tbl) +rb_imemo_fields_new_complex_tbl(VALUE owner, shape_id_t shape_id, st_table *tbl, bool shareable) { - VALUE fields = imemo_fields_new(owner, sizeof(struct rb_fields)); + VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable); IMEMO_OBJ_FIELDS(fields)->as.complex.table = tbl; + FL_SET_RAW(fields, OBJ_FIELD_HEAP); + // imemo fields objects should always have "RObject" layout. The + // layout in the shape describes the layout of the thing on which it is set. + // Imemo fields have the same layout as robject, therefore the layout + // should reflect that fact. + RBASIC_SET_SHAPE_ID(fields, rb_shape_id_with_robject_layout(shape_id)); st_foreach(tbl, imemo_fields_trigger_wb_i, (st_data_t)fields); return fields; } @@ -176,17 +200,18 @@ rb_imemo_fields_clone(VALUE fields_obj) shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj); VALUE clone; - if (rb_shape_too_complex_p(shape_id)) { - clone = rb_imemo_fields_new_complex(rb_imemo_fields_owner(fields_obj), 0); - RBASIC_SET_SHAPE_ID(clone, shape_id); + if (rb_shape_complex_p(shape_id)) { st_table *src_table = rb_imemo_fields_complex_tbl(fields_obj); - st_table *dest_table = rb_imemo_fields_complex_tbl(clone); + + st_table *dest_table = xcalloc(1, sizeof(st_table)); + clone = rb_imemo_fields_new_complex_tbl(rb_imemo_fields_owner(fields_obj), shape_id, dest_table, false /* TODO: check */); + st_replace(dest_table, src_table); + st_foreach(dest_table, imemo_fields_complex_wb_i, (st_data_t)clone); } else { - clone = imemo_fields_new(rb_imemo_fields_owner(fields_obj), RSHAPE_CAPACITY(shape_id)); - RBASIC_SET_SHAPE_ID(clone, shape_id); + clone = rb_imemo_fields_new(rb_imemo_fields_owner(fields_obj), shape_id, false /* TODO: check */); VALUE *fields = rb_imemo_fields_ptr(clone); attr_index_t fields_count = RSHAPE_LEN(shape_id); MEMCPY(fields, rb_imemo_fields_ptr(fields_obj), VALUE, fields_count); @@ -203,8 +228,8 @@ rb_imemo_fields_clear(VALUE fields_obj) { // When replacing an imemo/fields by another one, we must clear // its shape so that gc.c:obj_free_object_id won't be called. - if (rb_shape_obj_too_complex_p(fields_obj)) { - RBASIC_SET_SHAPE_ID(fields_obj, ROOT_TOO_COMPLEX_SHAPE_ID); + if (rb_obj_shape_complex_p(fields_obj)) { + RBASIC_SET_SHAPE_ID(fields_obj, ROOT_COMPLEX_SHAPE_ID); } else { RBASIC_SET_SHAPE_ID(fields_obj, ROOT_SHAPE_ID); @@ -213,6 +238,32 @@ rb_imemo_fields_clear(VALUE fields_obj) RBASIC_CLEAR_CLASS(fields_obj); } +VALUE +rb_imemo_subclasses_new(uint32_t capacity) +{ + size_t embed_size = offsetof(struct rb_subclasses, as) + capacity * sizeof(VALUE); + struct rb_subclasses *subs; + + if (rb_gc_size_allocatable_p(embed_size)) { + subs = (struct rb_subclasses *)rb_imemo_new(imemo_subclasses, 0, embed_size, true); + subs->count = 0; + subs->capacity = capacity; + memset(subs->as.embed, 0, capacity * sizeof(VALUE)); + rb_gc_declare_weak_references((VALUE)subs); + } + else { + subs = (struct rb_subclasses *)rb_imemo_new(imemo_subclasses, 0, sizeof(struct rb_subclasses), true); + subs->as.external = NULL; + subs->count = 0; + subs->capacity = 0; + FL_SET_RAW((VALUE)subs, IMEMO_SUBCLASSES_HEAP); + rb_gc_declare_weak_references((VALUE)subs); + subs->as.external = ZALLOC_N(VALUE, capacity); + subs->capacity = capacity; + } + return (VALUE)subs; +} + /* ========================================================================= * memsize * ========================================================================= */ @@ -251,16 +302,28 @@ rb_imemo_memsize(VALUE obj) case imemo_throw_data: break; case imemo_tmpbuf: - size += ((rb_imemo_tmpbuf_t *)obj)->cnt * sizeof(VALUE); + size += ((rb_imemo_tmpbuf_t *)obj)->size; break; + case imemo_cvar_entry: + break; case imemo_fields: - if (rb_shape_obj_too_complex_p(obj)) { + if (rb_obj_shape_complex_p(obj)) { size += st_memsize(IMEMO_OBJ_FIELDS(obj)->as.complex.table); } - else if (FL_TEST_RAW(obj, OBJ_FIELD_EXTERNAL)) { - size += RSHAPE_CAPACITY(RBASIC_SHAPE_ID(obj)) * sizeof(VALUE); + + break; + case imemo_subclasses: { + if (FL_TEST_RAW(obj, IMEMO_SUBCLASSES_HEAP)) { + struct rb_subclasses *subs = (struct rb_subclasses *)obj; + size += subs->capacity * sizeof(VALUE); } + + break; + } + case imemo_cdhash: + size += st_memsize(rb_imemo_cdhash_tbl(obj)) - sizeof(st_table); + break; default: rb_bug("unreachable"); @@ -308,9 +371,8 @@ mark_and_move_method_entry(rb_method_entry_t *ment, bool reference_updating) rb_gc_mark_and_move(&def->body.attr.location); break; case VM_METHOD_TYPE_BMETHOD: - rb_gc_mark_and_move(&def->body.bmethod.proc); - if (!reference_updating) { - if (def->body.bmethod.hooks) rb_hook_list_mark(def->body.bmethod.hooks); + if (!rb_gc_checking_shareable()) { + rb_gc_mark_and_move(&def->body.bmethod.proc); } break; case VM_METHOD_TYPE_ALIAS: @@ -378,7 +440,6 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating) RUBY_ASSERT(RB_TYPE_P(cc->klass, T_CLASS) || RB_TYPE_P(cc->klass, T_ICLASS)); RUBY_ASSERT(IMEMO_TYPE_P((VALUE)cc->cme_, imemo_ment)); - rb_gc_mark_weak((VALUE *)&cc->klass); if ((vm_cc_super_p(cc) || vm_cc_refinement_p(cc))) { rb_gc_mark_movable((VALUE)cc->cme_); } @@ -391,16 +452,27 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating) case imemo_constcache: { struct iseq_inline_constant_cache_entry *ice = (struct iseq_inline_constant_cache_entry *)obj; - rb_gc_mark_and_move(&ice->value); + if ((ice->flags & IMEMO_CONST_CACHE_SHAREABLE) || + !rb_gc_checking_shareable()) { + rb_gc_mark_and_move(&ice->value); + } break; } case imemo_cref: { rb_cref_t *cref = (rb_cref_t *)obj; - rb_gc_mark_and_move(&cref->klass_or_self); + if (!rb_gc_checking_shareable()) { + // cref->klass_or_self can be unshareable, but no way to access it from other ractors + rb_gc_mark_and_move(&cref->klass_or_self); + } + rb_gc_mark_and_move_ptr(&cref->next); - rb_gc_mark_and_move(&cref->refinements); + + // TODO: Ractor and refeinements are not resolved yet + if (!rb_gc_checking_shareable()) { + rb_gc_mark_and_move(&cref->refinements); + } break; } @@ -418,6 +490,13 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating) rb_gc_mark_and_move_ptr(&env->iseq); + if (VM_ENV_LOCAL_P(env->ep) && VM_ENV_BOXED_P(env->ep)) { + const rb_box_t *box = VM_ENV_BOX(env->ep); + if (BOX_USER_P(box)) { + rb_gc_mark_and_move((VALUE *)&box->box_object); + } + } + if (reference_updating) { ((VALUE *)env->ep)[VM_ENV_DATA_INDEX_ENV] = rb_gc_location(env->ep[VM_ENV_DATA_INDEX_ENV]); } @@ -448,8 +527,8 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating) rb_gc_mark_and_move((VALUE *)&memo->v1); rb_gc_mark_and_move((VALUE *)&memo->v2); - if (!reference_updating) { - rb_gc_mark_maybe(memo->u3.value); + if (FL_TEST_RAW(obj, MEMO_U3_IS_VALUE)) { + rb_gc_mark_and_move((VALUE *)&memo->u3.value); } break; @@ -478,31 +557,62 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating) const rb_imemo_tmpbuf_t *m = (const rb_imemo_tmpbuf_t *)obj; if (!reference_updating) { - do { - rb_gc_mark_locations(m->ptr, m->ptr + m->cnt); - } while ((m = m->next) != NULL); + rb_gc_mark_locations(m->ptr, m->ptr + (m->size / sizeof(VALUE))); } break; } + case imemo_cvar_entry: { + struct rb_cvar_class_tbl_entry *ent = (struct rb_cvar_class_tbl_entry *)obj; + rb_gc_mark_and_move(&ent->class_value); + rb_gc_mark_and_move((VALUE *)&ent->cref); + break; + } + case imemo_subclasses: { + if (reference_updating) { + struct rb_subclasses *subs = (struct rb_subclasses *)obj; + VALUE *entries = rb_imemo_subclasses_entries(obj); + for (uint32_t i = 0; i < subs->count; i++) { + if (entries[i]) { + entries[i] = rb_gc_location(entries[i]); + } + } + } + break; + } case imemo_fields: { rb_gc_mark_and_move((VALUE *)&RBASIC(obj)->klass); - if (rb_shape_obj_too_complex_p(obj)) { - st_table *tbl = rb_imemo_fields_complex_tbl(obj); - if (reference_updating) { - rb_gc_ref_update_table_values_only(tbl); + if (!rb_gc_checking_shareable()) { + // imemo_fields can refer unshareable objects + // even if the imemo_fields is shareable. + + if (rb_obj_shape_complex_p(obj)) { + st_table *tbl = rb_imemo_fields_complex_tbl(obj); + if (reference_updating) { + rb_gc_ref_update_table_values_only(tbl); + } + else { + rb_mark_tbl_no_pin(tbl); + } } else { - rb_mark_tbl_no_pin(tbl); + VALUE *fields = rb_imemo_fields_ptr(obj); + attr_index_t len = RSHAPE_LEN(RBASIC_SHAPE_ID(obj)); + for (attr_index_t i = 0; i < len; i++) { + rb_gc_mark_and_move(&fields[i]); + } } } + break; + } + case imemo_cdhash: { + st_table *tbl = rb_imemo_cdhash_tbl(obj); + if (reference_updating) { + rb_gc_update_set_refs(tbl); + } else { - VALUE *fields = rb_imemo_fields_ptr(obj); - attr_index_t len = RSHAPE_LEN(RBASIC_SHAPE_ID(obj)); - for (attr_index_t i = 0; i < len; i++) { - rb_gc_mark_and_move(&fields[i]); - } + rb_gc_mark_set_no_pin(tbl); } break; } @@ -519,7 +629,7 @@ static enum rb_id_table_iterator_result free_const_entry_i(VALUE value, void *data) { rb_const_entry_t *ce = (rb_const_entry_t *)value; - xfree(ce); + SIZED_FREE(ce); return ID_TABLE_CONTINUE; } @@ -533,12 +643,10 @@ rb_free_const_table(struct rb_id_table *tbl) static inline void imemo_fields_free(struct rb_fields *fields) { - if (rb_shape_obj_too_complex_p((VALUE)fields)) { + if (FL_TEST_RAW((VALUE)fields, OBJ_FIELD_HEAP)) { + RUBY_ASSERT(rb_shape_complex_p(RBASIC_SHAPE_ID((VALUE)fields))); st_free_table(fields->as.complex.table); } - else if (FL_TEST_RAW((VALUE)fields, OBJ_FIELD_EXTERNAL)) { - xfree(fields->as.external.ptr); - } } void @@ -553,8 +661,9 @@ rb_imemo_free(VALUE obj) const struct rb_callinfo *ci = ((const struct rb_callinfo *)obj); if (ci->kwarg) { - ((struct rb_callinfo_kwarg *)ci->kwarg)->references--; - if (ci->kwarg->references == 0) xfree((void *)ci->kwarg); + if (RUBY_ATOMIC_FETCH_SUB(((struct rb_callinfo_kwarg *)ci->kwarg)->references, 1) == 1) { + ruby_xfree_sized((void *)ci->kwarg, rb_callinfo_kwarg_bytes(ci->kwarg->keyword_len)); + } } RB_DEBUG_COUNTER_INC(obj_imemo_callinfo); @@ -572,7 +681,7 @@ rb_imemo_free(VALUE obj) rb_env_t *env = (rb_env_t *)obj; RUBY_ASSERT(VM_ENV_ESCAPED_P(env->ep)); - xfree((VALUE *)env->env); + SIZED_FREE_N(env->env, env->env_size); RB_DEBUG_COUNTER_INC(obj_imemo_env); break; @@ -603,13 +712,31 @@ rb_imemo_free(VALUE obj) break; case imemo_tmpbuf: - xfree(((rb_imemo_tmpbuf_t *)obj)->ptr); + ruby_xfree_sized(((rb_imemo_tmpbuf_t *)obj)->ptr, ((rb_imemo_tmpbuf_t *)obj)->size); RB_DEBUG_COUNTER_INC(obj_imemo_tmpbuf); break; + case imemo_cvar_entry: + RB_DEBUG_COUNTER_INC(obj_imemo_cvar_entry); + + break; case imemo_fields: imemo_fields_free(IMEMO_OBJ_FIELDS(obj)); RB_DEBUG_COUNTER_INC(obj_imemo_fields); + + break; + case imemo_subclasses: { + if (FL_TEST_RAW(obj, IMEMO_SUBCLASSES_HEAP)) { + struct rb_subclasses *subs = (struct rb_subclasses *)obj; + SIZED_FREE_N(subs->as.external, subs->capacity); + } + RB_DEBUG_COUNTER_INC(obj_imemo_subclasses); + break; + } + case imemo_cdhash: + st_free_embedded_table(rb_imemo_cdhash_tbl(obj)); + RB_DEBUG_COUNTER_INC(obj_imemo_cdhash); + break; default: rb_bug("unreachable"); |