diff options
author | Koichi Sasada <ko1@atdot.net> | 2021-11-13 02:12:20 +0900 |
---|---|---|
committer | Koichi Sasada <ko1@atdot.net> | 2021-11-15 15:58:56 +0900 |
commit | b1b73936c15fd490159a9b30ab50b8d5dfea1264 (patch) | |
tree | 23c8d640d3251ca0b3865087ae12b27941397d48 | |
parent | f9638c3b1716df4a94ea6ae0854cf55d66072ee4 (diff) |
`Primitive.mandatory_only?` for fast path
Compare with the C methods, A built-in methods written in Ruby is
slower if only mandatory parameters are given because it needs to
check the argumens and fill default values for optional and keyword
parameters (C methods can check the number of parameters with `argc`,
so there are no overhead). Passing mandatory arguments are common
(optional arguments are exceptional, in many cases) so it is important
to provide the fast path for such common cases.
`Primitive.mandatory_only?` is a special builtin function used with
`if` expression like that:
```ruby
def self.at(time, subsec = false, unit = :microsecond, in: nil)
if Primitive.mandatory_only?
Primitive.time_s_at1(time)
else
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
end
```
and it makes two ISeq,
```
def self.at(time, subsec = false, unit = :microsecond, in: nil)
Primitive.time_s_at(time, subsec, unit, Primitive.arg!(:in))
end
def self.at(time)
Primitive.time_s_at1(time)
end
```
and (2) is pointed by (1). Note that `Primitive.mandatory_only?`
should be used only in a condition of an `if` statement and the
`if` statement should be equal to the methdo body (you can not
put any expression before and after the `if` statement).
A method entry with `mandatory_only?` (`Time.at` on the above case)
is marked as `iseq_overload`. When the method will be dispatch only
with mandatory arguments (`Time.at(0)` for example), make another
method entry with ISeq (2) as mandatory only method entry and it
will be cached in an inline method cache.
The idea is similar discussed in https://bugs.ruby-lang.org/issues/16254
but it only checks mandatory parameters or more, because many cases
only mandatory parameters are given. If we find other cases (optional
or keyword parameters are used frequently and it hurts performance),
we can extend the feature.
Notes
Notes:
Merged: https://github.com/ruby/ruby/pull/5112
-rw-r--r-- | compile.c | 77 | ||||
-rw-r--r-- | gc.c | 14 | ||||
-rw-r--r-- | iseq.c | 4 | ||||
-rw-r--r-- | iseq.h | 1 | ||||
-rw-r--r-- | method.h | 6 | ||||
-rw-r--r-- | tool/mk_builtin_loader.rb | 5 | ||||
-rw-r--r-- | vm_callinfo.h | 19 | ||||
-rw-r--r-- | vm_core.h | 2 | ||||
-rw-r--r-- | vm_eval.c | 2 | ||||
-rw-r--r-- | vm_insnhelper.c | 19 | ||||
-rw-r--r-- | vm_method.c | 44 |
11 files changed, 177 insertions, 16 deletions
@@ -791,8 +791,10 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node) } case ISEQ_TYPE_METHOD: { + ISEQ_COMPILE_DATA(iseq)->root_node = node->nd_body; ADD_TRACE(ret, RUBY_EVENT_CALL); CHECK(COMPILE(ret, "scoped node", node->nd_body)); + ISEQ_COMPILE_DATA(iseq)->root_node = node->nd_body; ADD_TRACE(ret, RUBY_EVENT_RETURN); ISEQ_COMPILE_DATA(iseq)->last_line = nd_line(node); break; @@ -7884,6 +7886,65 @@ compile_builtin_arg(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, c UNKNOWN_NODE("arg!", node, COMPILE_NG); } +static NODE * +mandatory_node(const rb_iseq_t *iseq, const NODE *cond_node) +{ + const NODE *node = ISEQ_COMPILE_DATA(iseq)->root_node; + if (nd_type(node) == NODE_IF && node->nd_cond == cond_node) { + return node->nd_body; + } + else { + rb_bug("mandatory_node: can't find mandatory node"); + } +} + +static int +compile_builtin_mandatory_only_method(rb_iseq_t *iseq, const NODE *node, const NODE *line_node) +{ + // argumens + struct rb_args_info args = { + .pre_args_num = iseq->body->param.lead_num, + }; + NODE args_node; + rb_node_init(&args_node, NODE_ARGS, 0, 0, (VALUE)&args); + + // local table without non-mandatory parameters + const int skip_local_size = iseq->body->param.size - iseq->body->param.lead_num; + const int table_size = iseq->body->local_table_size - skip_local_size; + ID *tbl = ALLOCA_N(ID, table_size + 1); + tbl[0] = table_size; + int i; + + // lead parameters + for (i=0; i<iseq->body->param.lead_num; i++) { + tbl[i+1] = iseq->body->local_table[i]; + } + // local variables + for (; i<table_size; i++) { + tbl[i+1] = iseq->body->local_table[i + skip_local_size]; + } + + NODE scope_node; + rb_node_init(&scope_node, NODE_SCOPE, (VALUE)tbl, (VALUE)mandatory_node(iseq, node), (VALUE)&args_node); + + rb_ast_body_t ast = { + .root = &scope_node, + .compile_option = 0, + .script_lines = iseq->body->variable.script_lines, + }; + + int prev_inline_index = GET_VM()->builtin_inline_index; + + iseq->body->mandatory_only_iseq = + rb_iseq_new_with_opt(&ast, rb_iseq_base_label(iseq), + rb_iseq_path(iseq), rb_iseq_realpath(iseq), + INT2FIX(nd_line(line_node)), NULL, 0, + ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option); + + GET_VM()->builtin_inline_index = prev_inline_index; + return COMPILE_OK; +} + static int compile_builtin_function_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, const NODE *line_node, int popped, const rb_iseq_t *parent_block, LINK_ANCHOR *args, const char *builtin_func) @@ -7922,6 +7983,18 @@ compile_builtin_function_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NOD else if (strcmp("arg!", builtin_func) == 0) { return compile_builtin_arg(iseq, ret, args_node, line_node, popped); } + else if (strcmp("mandatory_only?", builtin_func) == 0) { + if (popped) { + rb_bug("mandatory_only? should be in if condition"); + } + else if (!LIST_INSN_SIZE_ZERO(ret)) { + rb_bug("mandatory_only? should be put on top"); + } + + ADD_INSN1(ret, line_node, putobject, Qfalse); + return compile_builtin_mandatory_only_method(iseq, node, line_node); + return COMPILE_OK; + } else if (1) { rb_bug("can't find builtin function:%s", builtin_func); } @@ -11628,6 +11701,7 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq) const ibf_offset_t catch_table_offset = ibf_dump_catch_table(dump, iseq); const int parent_iseq_index = ibf_dump_iseq(dump, iseq->body->parent_iseq); const int local_iseq_index = ibf_dump_iseq(dump, iseq->body->local_iseq); + const int mandatory_only_iseq_index = ibf_dump_iseq(dump, iseq->body->mandatory_only_iseq); const ibf_offset_t ci_entries_offset = ibf_dump_ci_entries(dump, iseq); const ibf_offset_t outer_variables_offset = ibf_dump_outer_variables(dump, iseq); @@ -11690,6 +11764,7 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq) ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(catch_table_offset)); ibf_dump_write_small_value(dump, parent_iseq_index); ibf_dump_write_small_value(dump, local_iseq_index); + ibf_dump_write_small_value(dump, mandatory_only_iseq_index); ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(ci_entries_offset)); ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(outer_variables_offset)); ibf_dump_write_small_value(dump, body->variable.flip_count); @@ -11797,6 +11872,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset) const ibf_offset_t catch_table_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos)); const int parent_iseq_index = (int)ibf_load_small_value(load, &reading_pos); const int local_iseq_index = (int)ibf_load_small_value(load, &reading_pos); + const int mandatory_only_iseq_index = (int)ibf_load_small_value(load, &reading_pos); const ibf_offset_t ci_entries_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos)); const ibf_offset_t outer_variables_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos)); const rb_snum_t variable_flip_count = (rb_snum_t)ibf_load_small_value(load, &reading_pos); @@ -11859,6 +11935,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset) load_body->catch_table = ibf_load_catch_table(load, catch_table_offset, catch_table_size); load_body->parent_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)parent_iseq_index); load_body->local_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)local_iseq_index); + load_body->mandatory_only_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)mandatory_only_iseq_index); ibf_load_code(load, iseq, bytecode_offset, bytecode_size, iseq_size); #if VM_INSN_INFO_TABLE_IMPL == 2 @@ -2960,7 +2960,7 @@ cc_table_mark_i(ID id, VALUE ccs_ptr, void *data_ptr) for (int i=0; i<ccs->len; i++) { VM_ASSERT(data->klass == ccs->entries[i].cc->klass); - VM_ASSERT(ccs->cme == vm_cc_cme(ccs->entries[i].cc)); + VM_ASSERT(vm_cc_check_cme(ccs->entries[i].cc, ccs->cme)); gc_mark(data->objspace, (VALUE)ccs->entries[i].ci); gc_mark(data->objspace, (VALUE)ccs->entries[i].cc); @@ -6443,10 +6443,11 @@ mark_method_entry(rb_objspace_t *objspace, const rb_method_entry_t *me) if (def) { switch (def->type) { - case VM_METHOD_TYPE_ISEQ: - if (def->body.iseq.iseqptr) gc_mark(objspace, (VALUE)def->body.iseq.iseqptr); - gc_mark(objspace, (VALUE)def->body.iseq.cref); - break; + case VM_METHOD_TYPE_ISEQ: + if (def->body.iseq.iseqptr) gc_mark(objspace, (VALUE)def->body.iseq.iseqptr); + gc_mark(objspace, (VALUE)def->body.iseq.cref); + if (def->body.iseq.mandatory_only_cme) gc_mark(objspace, (VALUE)def->body.iseq.mandatory_only_cme); + break; case VM_METHOD_TYPE_ATTRSET: case VM_METHOD_TYPE_IVAR: gc_mark(objspace, def->body.attr.location); @@ -9612,6 +9613,9 @@ gc_ref_update_method_entry(rb_objspace_t *objspace, rb_method_entry_t *me) TYPED_UPDATE_IF_MOVED(objspace, rb_iseq_t *, def->body.iseq.iseqptr); } TYPED_UPDATE_IF_MOVED(objspace, rb_cref_t *, def->body.iseq.cref); + if (def->body.iseq.mandatory_only_cme) { + TYPED_UPDATE_IF_MOVED(objspace, rb_callable_method_entry_t *, def->body.iseq.mandatory_only_cme); + } break; case VM_METHOD_TYPE_ATTRSET: case VM_METHOD_TYPE_IVAR: @@ -272,6 +272,9 @@ rb_iseq_update_references(rb_iseq_t *iseq) if (body->parent_iseq) { body->parent_iseq = (struct rb_iseq_struct *)rb_gc_location((VALUE)body->parent_iseq); } + if (body->mandatory_only_iseq) { + body->mandatory_only_iseq = (struct rb_iseq_struct *)rb_gc_location((VALUE)body->mandatory_only_iseq); + } if (body->call_data) { for (unsigned int i=0; i<body->ci_size; i++) { struct rb_call_data *cds = body->call_data; @@ -351,6 +354,7 @@ rb_iseq_mark(const rb_iseq_t *iseq) rb_gc_mark_movable(body->location.label); rb_gc_mark_movable(body->location.base_label); rb_gc_mark_movable(body->location.pathobj); + RUBY_MARK_MOVABLE_UNLESS_NULL((VALUE)body->mandatory_only_iseq); RUBY_MARK_MOVABLE_UNLESS_NULL((VALUE)body->parent_iseq); if (body->call_data) { @@ -117,6 +117,7 @@ struct iseq_compile_data { const rb_compile_option_t *option; struct rb_id_table *ivar_cache_table; const struct rb_builtin_function *builtin_function_table; + const NODE *root_node; #if OPT_SUPPORT_JOKE st_table *labels_table; #endif @@ -132,8 +132,9 @@ typedef struct rb_iseq_struct rb_iseq_t; #endif typedef struct rb_method_iseq_struct { - rb_iseq_t * iseqptr; /*!< iseq pointer, should be separated from iseqval */ + const rb_iseq_t * iseqptr; /*!< iseq pointer, should be separated from iseqval */ rb_cref_t * cref; /*!< class reference, should be marked */ + const rb_callable_method_entry_t *mandatory_only_cme; } rb_method_iseq_t; /* check rb_add_method_iseq() when modify the fields */ typedef struct rb_method_cfunc_struct { @@ -171,7 +172,8 @@ enum method_optimized_type { struct rb_method_definition_struct { BITFIELD(rb_method_type_t, type, VM_METHOD_TYPE_MINIMUM_BITS); - int alias_count : 28; + unsigned int iseq_overload: 1; + int alias_count : 27; int complemented_count : 28; union { diff --git a/tool/mk_builtin_loader.rb b/tool/mk_builtin_loader.rb index 6740ec5c17..02941735f7 100644 --- a/tool/mk_builtin_loader.rb +++ b/tool/mk_builtin_loader.rb @@ -100,6 +100,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil when :call, :command_call # CALL _, recv, sep, mid, (_, args) = tree end + if mid raise "unknown sexp: #{mid.inspect}" unless %i[@ident @const].include?(mid.first) _, mid, (lineno,) = mid @@ -126,7 +127,7 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil args.pop unless (args ||= []).last argc = args.size - if /(.+)\!\z/ =~ func_name + if /(.+)[\!\?]\z/ =~ func_name case $1 when 'attr' text = inline_text(argc, args.first) @@ -156,6 +157,8 @@ def collect_builtin base, tree, name, bs, inlines, locals = nil func_name = nil # required inlines[inlines.size] = [lineno, text, nil, nil] argc -= 1 + when 'mandatory_only' + func_name = nil when 'arg' argc == 1 or raise "unexpected argument number #{argc}" (arg = args.first)[0] == :symbol_literal or raise "symbol literal expected #{args}" diff --git a/vm_callinfo.h b/vm_callinfo.h index d5f4388fa8..4f3518c066 100644 --- a/vm_callinfo.h +++ b/vm_callinfo.h @@ -446,6 +446,25 @@ vm_ccs_p(const struct rb_class_cc_entries *ccs) { return ccs->debug_sig == ~(VALUE)ccs; } + +static inline bool +vm_cc_check_cme(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme) +{ + if (vm_cc_cme(cc) == cme || + (cme->def->iseq_overload && vm_cc_cme(cc) == cme->def->body.iseq.mandatory_only_cme)) { + return true; + } + else { +#if 1 + fprintf(stderr, "iseq_overload:%d mandatory_only_cme:%p eq:%d\n", + (int)cme->def->iseq_overload, + (void *)cme->def->body.iseq.mandatory_only_cme, + vm_cc_cme(cc) == cme->def->body.iseq.mandatory_only_cme); +#endif + return false; + } +} + #endif // gc.c @@ -482,6 +482,8 @@ struct rb_iseq_constant_body { bool builtin_inline_p; struct rb_id_table *outer_variables; + const rb_iseq_t *mandatory_only_iseq; + #if USE_MJIT /* The following fields are MJIT related info. */ VALUE (*jit_func)(struct rb_execution_context_struct *, @@ -455,7 +455,7 @@ gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, int argc) if (LIKELY(!METHOD_ENTRY_INVALIDATED(cme) && cme->called_id == mid)) { - VM_ASSERT(vm_cc_cme(cc) == rb_callable_method_entry(klass, mid)); + VM_ASSERT(vm_cc_check_cme(cc, rb_callable_method_entry(klass, mid))); RB_DEBUG_COUNTER_INC(gccct_hit); return cc; diff --git a/vm_insnhelper.c b/vm_insnhelper.c index b9950f4fe2..8e0793e070 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -678,7 +678,7 @@ rb_vm_frame_method_entry(const rb_control_frame_t *cfp) return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE); } -static rb_iseq_t * +static const rb_iseq_t * method_entry_iseqptr(const rb_callable_method_entry_t *me) { switch (me->def->type) { @@ -1754,13 +1754,16 @@ vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass) VM_ASSERT(vm_ci_mid(ci) == mid); VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); VM_ASSERT(vm_cc_class_check(cc, klass)); - VM_ASSERT(vm_cc_cme(cc) == ccs->cme); + VM_ASSERT(vm_cc_check_cme(cc, ccs->cme)); } return TRUE; } #endif #ifndef MJIT_HEADER + +static const rb_callable_method_entry_t *overloaded_cme(const rb_callable_method_entry_t *cme); + static const struct rb_callcache * vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci) { @@ -1829,7 +1832,6 @@ vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci) VM_ASSERT(cme == rb_callable_method_entry(klass, mid)); - const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general); METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme); if (ccs == NULL) { @@ -1846,6 +1848,14 @@ vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci) } } + if ((cme->def->iseq_overload && + (int)vm_ci_argc(ci) == method_entry_iseqptr(cme)->body->param.lead_num)) { + // use alternative + cme = overloaded_cme(cme); + METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme); + // rp(cme); + } + const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general); vm_ccs_push(klass, ccs, ci, cc); VM_ASSERT(vm_cc_cme(cc) != NULL); @@ -3529,9 +3539,10 @@ vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, st { const struct rb_callinfo *ci = calling->ci; const struct rb_callcache *cc = calling->cc; + const rb_callable_method_entry_t *cme = vm_cc_cme(cc); VALUE v; - switch (vm_cc_cme(cc)->def->type) { + switch (cme->def->type) { case VM_METHOD_TYPE_ISEQ: CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE); return vm_call_iseq_setup(ec, cfp, calling); diff --git a/vm_method.c b/vm_method.c index 75f38d386d..b41b0ff908 100644 --- a/vm_method.c +++ b/vm_method.c @@ -208,6 +208,10 @@ clear_method_cache_by_id_in_class(VALUE klass, ID mid) vm_cme_invalidate((rb_callable_method_entry_t *)cme); RB_DEBUG_COUNTER_INC(cc_invalidate_tree_cme); + + if (cme->def->iseq_overload) { + vm_cme_invalidate((rb_callable_method_entry_t *)cme->def->body.iseq.mandatory_only_cme); + } } // invalidate complement tbl @@ -451,10 +455,13 @@ rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *de case VM_METHOD_TYPE_ISEQ: { rb_method_iseq_t *iseq_body = (rb_method_iseq_t *)opts; + const rb_iseq_t *iseq = iseq_body->iseqptr; rb_cref_t *method_cref, *cref = iseq_body->cref; /* setup iseq first (before invoking GC) */ - RB_OBJ_WRITE(me, &def->body.iseq.iseqptr, iseq_body->iseqptr); + RB_OBJ_WRITE(me, &def->body.iseq.iseqptr, iseq); + + if (iseq->body->mandatory_only_iseq) def->iseq_overload = 1; if (0) vm_cref_dump("rb_method_definition_create", cref); @@ -531,7 +538,8 @@ method_definition_reset(const rb_method_entry_t *me) case VM_METHOD_TYPE_ISEQ: RB_OBJ_WRITTEN(me, Qundef, def->body.iseq.iseqptr); RB_OBJ_WRITTEN(me, Qundef, def->body.iseq.cref); - break; + RB_OBJ_WRITTEN(me, Qundef, def->body.iseq.mandatory_only_cme); + break; case VM_METHOD_TYPE_ATTRSET: case VM_METHOD_TYPE_IVAR: RB_OBJ_WRITTEN(me, Qundef, def->body.attr.location); @@ -893,6 +901,35 @@ rb_method_entry_make(VALUE klass, ID mid, VALUE defined_class, rb_method_visibil return me; } +static const rb_callable_method_entry_t * +overloaded_cme(const rb_callable_method_entry_t *cme) +{ + VM_ASSERT(cme->def->iseq_overload); + VM_ASSERT(cme->def->type == VM_METHOD_TYPE_ISEQ); + VM_ASSERT(cme->def->body.iseq.iseqptr != NULL); + + const rb_callable_method_entry_t *monly_cme = cme->def->body.iseq.mandatory_only_cme; + + if (monly_cme && !METHOD_ENTRY_INVALIDATED(monly_cme)) { + // ok + } + else { + rb_method_definition_t *def = rb_method_definition_create(VM_METHOD_TYPE_ISEQ, cme->def->original_id); + def->body.iseq.cref = cme->def->body.iseq.cref; + def->body.iseq.iseqptr = cme->def->body.iseq.iseqptr->body->mandatory_only_iseq; + + rb_method_entry_t *me = rb_method_entry_alloc(cme->called_id, + cme->owner, + cme->defined_class, + def); + METHOD_ENTRY_VISI_SET(me, METHOD_ENTRY_VISI(cme)); + RB_OBJ_WRITE(cme, &cme->def->body.iseq.mandatory_only_cme, me); + monly_cme = (rb_callable_method_entry_t *)me; + } + + return monly_cme; +} + #define CALL_METHOD_HOOK(klass, hook, mid) do { \ const VALUE arg = ID2SYM(mid); \ VALUE recv_class = (klass); \ @@ -932,6 +969,7 @@ rb_add_method_iseq(VALUE klass, ID mid, const rb_iseq_t *iseq, rb_cref_t *cref, iseq_body.iseqptr = iseq; iseq_body.cref = cref; + rb_add_method(klass, mid, VM_METHOD_TYPE_ISEQ, &iseq_body, visi); } @@ -1876,7 +1914,7 @@ rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_defini switch (d1->type) { case VM_METHOD_TYPE_ISEQ: - return d1->body.iseq.iseqptr == d2->body.iseq.iseqptr; + return d1->body.iseq.iseqptr == d2->body.iseq.iseqptr; case VM_METHOD_TYPE_CFUNC: return d1->body.cfunc.func == d2->body.cfunc.func && |