diff options
Diffstat (limited to 'insns.def')
| -rw-r--r-- | insns.def | 882 |
1 files changed, 587 insertions, 295 deletions
@@ -109,14 +109,14 @@ getblockparam VM_ASSERT(VM_ENV_LOCAL_P(ep)); if (!VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)) { - val = rb_vm_bh_to_procval(ec, VM_ENV_BLOCK_HANDLER(ep)); - vm_env_write(ep, -(int)idx, val); - VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM); + val = rb_vm_bh_to_procval(ec, VM_ENV_BLOCK_HANDLER(ep)); + vm_env_write(ep, -(int)idx, val); + VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM); } else { - val = *(ep - idx); - RB_DEBUG_COUNTER_INC(lvar_get); - (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0); + val = *(ep - idx); + RB_DEBUG_COUNTER_INC(lvar_get); + (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0); } } @@ -145,40 +145,41 @@ getblockparamproxy (lindex_t idx, rb_num_t level) () (VALUE val) +// attr bool zjit_profile = true; { const VALUE *ep = vm_get_ep(GET_EP(), level); VM_ASSERT(VM_ENV_LOCAL_P(ep)); if (!VM_ENV_FLAGS(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM)) { - VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep); - - if (block_handler) { - switch (vm_block_handler_type(block_handler)) { - case block_handler_type_iseq: - case block_handler_type_ifunc: - val = rb_block_param_proxy; - break; - case block_handler_type_symbol: - val = rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler)); - goto INSN_LABEL(set); - case block_handler_type_proc: - val = VM_BH_TO_PROC(block_handler); - goto INSN_LABEL(set); - default: - VM_UNREACHABLE(getblockparamproxy); - } - } - else { - val = Qnil; - INSN_LABEL(set): - vm_env_write(ep, -(int)idx, val); - VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM); - } + VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep); + + if (block_handler) { + switch (vm_block_handler_type(block_handler)) { + case block_handler_type_iseq: + case block_handler_type_ifunc: + val = rb_block_param_proxy; + break; + case block_handler_type_symbol: + val = rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler)); + goto INSN_LABEL(set); + case block_handler_type_proc: + val = VM_BH_TO_PROC(block_handler); + goto INSN_LABEL(set); + default: + VM_UNREACHABLE(getblockparamproxy); + } + } + else { + val = Qnil; + INSN_LABEL(set): + vm_env_write(ep, -(int)idx, val); + VM_ENV_FLAGS_SET(ep, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM); + } } else { - val = *(ep - idx); - RB_DEBUG_COUNTER_INC(lvar_get); - (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0); + val = *(ep - idx); + RB_DEBUG_COUNTER_INC(lvar_get); + (void)RB_DEBUG_COUNTER_INC_IF(lvar_get_dynamic, level > 0); } } @@ -207,64 +208,77 @@ setspecial /* Get value of instance variable id of self. */ DEFINE_INSN getinstancevariable -(ID id, IC ic) +(ID id, IVC ic) () (VALUE val) -/* "instance variable not initialized" warning can be hooked. */ -// attr bool leaf = false; /* has rb_warning() */ +/* Ractor crashes when it accesses class/module-level instances variables. */ +// attr bool leaf = false; /* has IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR() */ +// attr bool zjit_profile = true; { - val = vm_getinstancevariable(GET_SELF(), id, ic); + val = vm_getinstancevariable(GET_ISEQ(), GET_SELF(), id, ic); } /* Set value of instance variable id of self to val. */ DEFINE_INSN setinstancevariable -(ID id, IC ic) +(ID id, IVC ic) (VALUE val) () -// attr bool leaf = false; /* has rb_check_frozen_internal() */ +// attr bool leaf = false; /* has rb_check_frozen() */ +// attr bool zjit_profile = true; { - vm_setinstancevariable(GET_SELF(), id, val, ic); + vm_setinstancevariable(GET_ISEQ(), GET_SELF(), id, val, ic); } /* Get value of class variable id of klass as val. */ DEFINE_INSN getclassvariable -(ID id) +(ID id, ICVARC ic) () (VALUE val) /* "class variable access from toplevel" warning can be hooked. */ // attr bool leaf = false; /* has rb_warning() */ { - val = rb_cvar_get(vm_get_cvar_base(vm_get_cref(GET_EP()), GET_CFP()), id); + rb_control_frame_t *cfp = GET_CFP(); + val = vm_getclassvariable(GET_ISEQ(), cfp, id, ic); } /* Set value of class variable id of klass as val. */ DEFINE_INSN setclassvariable -(ID id) +(ID id, ICVARC ic) (VALUE val) () /* "class variable access from toplevel" warning can be hooked. */ // attr bool leaf = false; /* has rb_warning() */ { vm_ensure_not_refinement_module(GET_SELF()); - rb_cvar_set(vm_get_cvar_base(vm_get_cref(GET_EP()), GET_CFP()), id, val); + vm_setclassvariable(GET_ISEQ(), GET_CFP(), id, val, ic); } -/* Get constant variable id. If klass is Qnil, constants +DEFINE_INSN +opt_getconstant_path +(IC ic) +() +(VALUE val) +// attr bool leaf = false; /* may autoload or raise */ +{ + val = rb_vm_opt_getconstant_path(ec, GET_CFP(), ic); +} + +/* Get constant variable id. If klass is Qnil and allow_nil is Qtrue, constants are searched in the current scope. Otherwise, get constant under klass class or module. */ DEFINE_INSN getconstant (ID id) -(VALUE klass) +(VALUE klass, VALUE allow_nil) (VALUE val) /* getconstant can kick autoload */ // attr bool leaf = false; /* has rb_autoload_load() */ { - val = vm_get_ev_const(ec, klass, id, 0); + val = vm_get_ev_const(ec, klass, id, allow_nil == Qtrue, 0); } /* Set constant variable id under cbase class or module. @@ -288,25 +302,23 @@ setconstant /* get global variable id. */ DEFINE_INSN getglobal -(GENTRY entry) +(ID gid) () (VALUE val) -// attr bool leaf = leafness_of_getglobal(entry); +// attr bool leaf = false; { - struct rb_global_entry *gentry = (void *)entry; - val = rb_gvar_get(gentry); + val = rb_gvar_get(gid); } /* set global variable id as val. */ DEFINE_INSN setglobal -(GENTRY entry) +(ID gid) (VALUE val) () -// attr bool leaf = leafness_of_setglobal(entry); +// attr bool leaf = false; { - struct rb_global_entry *gentry = (void *)entry; - rb_gvar_set(gentry, val); + rb_gvar_set(gid, val); } /**********************************************************/ @@ -351,6 +363,7 @@ putspecialobject (rb_num_t value_type) () (VALUE val) +// attr bool leaf = (value_type == VM_SPECIAL_OBJECT_VMCORE); /* others may raise when allocating singleton */ { enum vm_special_object_type type; @@ -360,12 +373,22 @@ putspecialobject /* put string val. string will be copied. */ DEFINE_INSN -putstring +dupstring +(VALUE str) +() +(VALUE val) +{ + val = rb_ec_str_resurrect(ec, str, false); +} + +/* put chilled string val. string will be copied but frozen in the future. */ +DEFINE_INSN +dupchilledstring (VALUE str) () (VALUE val) { - val = rb_str_resurrect(str); + val = rb_ec_str_resurrect(ec, str, true); } /* put concatenate strings */ @@ -375,16 +398,17 @@ concatstrings (...) (VALUE val) /* This instruction can concat UTF-8 and binary strings, resulting in - * Encoding::CompatiblityError. */ + * Encoding::CompatibilityError. */ // attr bool leaf = false; /* has rb_enc_cr_str_buf_cat() */ // attr rb_snum_t sp_inc = 1 - (rb_snum_t)num; { val = rb_str_concat_literals(num, STACK_ADDR_FROM_TOP(num)); } -/* push the result of to_s. */ +/* Convert the result to string if not already a string. + This is used as a backup if to_s does not return a string. */ DEFINE_INSN -tostring +anytostring () (VALUE val, VALUE str) (VALUE val) @@ -392,16 +416,6 @@ tostring val = rb_obj_as_string_result(str, val); } -/* Freeze (dynamically) created strings. if debug_info is given, set it. */ -DEFINE_INSN -freezestring -(VALUE debug_info) -(VALUE str) -(VALUE str) -{ - vm_freezestring(str, debug_info); -} - /* compile str to Regexp and push it. opt is the option for the Regexp. */ @@ -415,9 +429,7 @@ toregexp // attr bool leaf = false; // attr rb_snum_t sp_inc = 1 - (rb_snum_t)cnt; { - const VALUE ary = rb_ary_tmp_new_from_values(0, cnt, STACK_ADDR_FROM_TOP(cnt)); - val = rb_reg_new_ary(ary, (int)opt); - rb_ary_clear(ary); + val = rb_reg_new_from_values(cnt, STACK_ADDR_FROM_TOP(cnt), (int)opt); } /* intern str to Symbol and push it. */ @@ -438,7 +450,21 @@ newarray (VALUE val) // attr rb_snum_t sp_inc = 1 - (rb_snum_t)num; { - val = rb_ary_new4(num, STACK_ADDR_FROM_TOP(num)); + val = rb_ec_ary_new_from_values(ec, num, STACK_ADDR_FROM_TOP(num)); +} + +/* push hash onto array unless the hash is empty (as empty keyword + splats should be ignored). + */ +DEFINE_INSN +pushtoarraykwsplat +() +(VALUE ary, VALUE hash) +(VALUE ary) +{ + if (!RHASH_EMPTY_P(hash)) { + rb_ary_push(ary, hash); + } } /* dup array */ @@ -477,13 +503,16 @@ expandarray (rb_num_t num, rb_num_t flag) (..., VALUE ary) (...) +// attr bool handles_sp = true; // attr bool leaf = false; /* has rb_check_array_type() */ // attr rb_snum_t sp_inc = (rb_snum_t)num - 1 + (flag & 1 ? 1 : 0); { - vm_expandarray(GET_SP(), ary, num, (int)flag); + vm_expandarray(GET_CFP(), ary, num, (int)flag); } -/* concat two arrays */ +/* concat two arrays, without modifying first array. + * attempts to convert both objects to arrays using to_a. + */ DEFINE_INSN concatarray () @@ -494,6 +523,32 @@ concatarray ary = vm_concat_array(ary1, ary2); } +/* concat second array to first array. + * first argument must already be an array. + * attempts to convert second object to array using to_a. + */ +DEFINE_INSN +concattoarray +() +(VALUE ary1, VALUE ary2) +(VALUE ary) +// attr bool leaf = false; /* has rb_check_array_type() */ +{ + ary = vm_concat_to_array(ary1, ary2); +} + +/* push given number of objects to array directly before. */ +DEFINE_INSN +pushtoarray +(rb_num_t num) +(...) +(VALUE val) +// attr rb_snum_t sp_inc = -(rb_snum_t)num; +{ + const VALUE *objp = STACK_ADDR_FROM_TOP(num); + val = rb_ary_cat(*(objp-1), objp, num); +} + /* call to_a on array ary to splat */ DEFINE_INSN splatarray @@ -505,6 +560,23 @@ splatarray obj = vm_splat_array(flag, ary); } +/* call to_hash on hash to keyword splat before converting block */ +DEFINE_INSN +splatkw +() +(VALUE hash, VALUE block) +(VALUE obj, VALUE block) +// attr bool leaf = false; /* has rb_to_hash_type() */ +// attr bool zjit_profile = true; +{ + if (NIL_P(hash)) { + obj = Qnil; + } + else { + obj = rb_to_hash_type(hash); + } +} + /* put new Hash from n elements. n must be an even number. */ DEFINE_INSN newhash @@ -516,10 +588,11 @@ newhash { RUBY_DTRACE_CREATE_HOOK(HASH, num); - val = rb_hash_new_with_size(num / 2); - if (num) { - rb_hash_bulk_insert(num, STACK_ADDR_FROM_TOP(num), val); + val = rb_hash_new_with_bulk_insert(num, STACK_ADDR_FROM_TOP(num)); + } + else { + val = rb_hash_new(); } } @@ -586,7 +659,7 @@ swap /* reverse stack top N order. */ DEFINE_INSN -reverse +opt_reverse (rb_num_t n) (...) (...) @@ -604,7 +677,7 @@ reverse } /* for stack caching. */ -DEFINE_INSN +DEFINE_INSN_IF(STACK_CACHING) reput () (..., VALUE val) @@ -654,19 +727,37 @@ adjuststack /* defined? */ DEFINE_INSN defined -(rb_num_t op_type, VALUE obj, VALUE needstr) +(rb_num_t op_type, VALUE obj, VALUE pushval) (VALUE v) (VALUE val) // attr bool leaf = leafness_of_defined(op_type); { - val = vm_defined(ec, GET_CFP(), op_type, obj, needstr, v); + val = Qnil; + if (vm_defined(ec, GET_CFP(), op_type, obj, v)) { + val = pushval; + } +} + +/* defined?(@foo) */ +DEFINE_INSN +definedivar +(ID id, IVC ic, VALUE pushval) +() +(VALUE val) +// attr bool leaf = false; +// attr bool zjit_profile = true; +{ + val = Qnil; + if (!UNDEF_P(vm_getivar(GET_SELF(), id, GET_ISEQ(), ic, NULL, FALSE, Qundef))) { + val = pushval; + } } /* check `target' matches `pattern'. `flag & VM_CHECKMATCH_TYPE_MASK' describe how to check pattern. VM_CHECKMATCH_TYPE_WHEN: ignore target and check pattern is truthy. VM_CHECKMATCH_TYPE_CASE: check `patten === target'. - VM_CHECKMATCH_TYPE_RESCUE: check `pattern.kind_op?(Module) && pattern === target'. + VM_CHECKMATCH_TYPE_RESCUE: check `pattern.kind_of?(Module) && pattern === target'. if `flag & VM_CHECKMATCH_ARRAY' is not 0, then `patten' is array of patterns. */ DEFINE_INSN @@ -696,17 +787,7 @@ checktype (VALUE val) (VALUE ret) { - ret = (TYPE(val) == (int)type) ? Qtrue : Qfalse; -} - -/* get method reference. */ -DEFINE_INSN -methodref -(ID id) -(VALUE val) -(VALUE ret) -{ - ret = rb_obj_method(val, ID2SYM(id)); + ret = RBOOL(TYPE(val) == (int)type); } /**********************************************************/ @@ -714,7 +795,7 @@ methodref /**********************************************************/ /* enter class definition scope. if super is Qfalse, and class - "klass" is defined, it's redefine. otherwise, define "klass" class. + "klass" is defined, it's redefined. Otherwise, define "klass" class. */ DEFINE_INSN defineclass @@ -723,16 +804,23 @@ defineclass (VALUE val) { VALUE klass = vm_find_or_create_class_by_id(id, flags, cbase, super); + const rb_box_t *box = rb_current_box(); rb_iseq_check(class_iseq); + rb_cref_t *cref = vm_cref_push(ec, klass, NULL, FALSE, FALSE); + + if (VM_DEFINECLASS_DYNAMIC_CREF_P(flags)) { + CREF_DYNAMIC_SET(cref); + } + /* enter scope */ vm_push_frame(ec, class_iseq, VM_FRAME_MAGIC_CLASS | VM_ENV_FLAG_LOCAL, klass, - GET_BLOCK_HANDLER(), - (VALUE)vm_cref_push(ec, klass, NULL, FALSE), - class_iseq->body->iseq_encoded, GET_SP(), - class_iseq->body->local_table_size, - class_iseq->body->stack_max); + GC_GUARDED_PTR(box), + (VALUE)cref, + ISEQ_BODY(class_iseq)->iseq_encoded, GET_SP(), + ISEQ_BODY(class_iseq)->local_table_size, + ISEQ_BODY(class_iseq)->stack_max); RESTORE_REGS(); NEXT_INSN(); } @@ -762,15 +850,45 @@ definesmethod /* invoke method. */ DEFINE_INSN send -(CALL_INFO ci, CALL_CACHE cc, ISEQ blockiseq) +(CALL_DATA cd, ISEQ blockiseq) +(...) +(VALUE val) +// attr bool zjit_profile = true; +// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); +{ + VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false); + val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method); + JIT_EXEC(ec, val); + + if (UNDEF_P(val)) { + RESTORE_REGS(); + NEXT_INSN(); + } +} + +/* invoke forward method. */ +DEFINE_INSN +sendforward +(CALL_DATA cd, ISEQ blockiseq) (...) (VALUE val) -// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); +// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); { - VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), ci, blockiseq, false); - val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_method_wrap); + struct rb_forwarding_call_data adjusted_cd; + struct rb_callinfo adjusted_ci; + + VALUE bh = vm_caller_setup_fwd_args(ec, GET_CFP(), cd, blockiseq, 0, &adjusted_cd, &adjusted_ci); - if (val == Qundef) { + val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method); + JIT_EXEC(ec, val); + + if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { + RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + } + + if (UNDEF_P(val)) { RESTORE_REGS(); NEXT_INSN(); } @@ -779,52 +897,164 @@ send /* Invoke method without block */ DEFINE_INSN opt_send_without_block -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (...) (VALUE val) +// attr bool zjit_profile = true; // attr bool handles_sp = true; -// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); +// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); { VALUE bh = VM_BLOCK_HANDLER_NONE; - val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_method_wrap); + val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method); + JIT_EXEC(ec, val); - if (val == Qundef) { + if (UNDEF_P(val)) { RESTORE_REGS(); NEXT_INSN(); } } +/* Jump if "new" method has been defined by user */ +DEFINE_INSN +opt_new +(CALL_DATA cd, OFFSET dst) +() +() +// attr bool leaf = false; +{ + VALUE argc = vm_ci_argc(cd->ci); + VALUE val = TOPN(argc); + + // The bookkeeping slot should be empty. + RUBY_ASSERT(TOPN(argc + 1) == Qnil); + + if (vm_method_cfunc_is(GET_CFP(), cd, val, rb_class_new_instance_pass_kw)) { + RB_DEBUG_COUNTER_INC(opt_new_hit); + val = rb_obj_alloc(val); + TOPN(argc) = val; + TOPN(argc + 1) = val; + } + else { + RB_DEBUG_COUNTER_INC(opt_new_miss); + JUMP(dst); + } +} + +/* Convert object to string using to_s or equivalent. */ +DEFINE_INSN +objtostring +(CALL_DATA cd) +(VALUE recv) +(VALUE val) +// attr bool leaf = false; +// attr bool zjit_profile = true; +{ + val = vm_objtostring(GET_CFP(), recv, cd); + + if (UNDEF_P(val)) { + CALL_SIMPLE_METHOD(); + } +} + +DEFINE_INSN +opt_ary_freeze +(VALUE ary, CALL_DATA cd) +() +(VALUE val) +{ + val = vm_opt_ary_freeze(ary, BOP_FREEZE, idFreeze); + + if (UNDEF_P(val)) { + RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary)); + PUSH(rb_ary_resurrect(ary)); + CALL_SIMPLE_METHOD(); + } +} + +DEFINE_INSN +opt_hash_freeze +(VALUE hash, CALL_DATA cd) +() +(VALUE val) +{ + val = vm_opt_hash_freeze(hash, BOP_FREEZE, idFreeze); + + if (UNDEF_P(val)) { + RUBY_DTRACE_CREATE_HOOK(HASH, RHASH_SIZE(hash) << 1); + PUSH(rb_hash_resurrect(hash)); + CALL_SIMPLE_METHOD(); + } +} + DEFINE_INSN opt_str_freeze -(VALUE str, CALL_INFO ci, CALL_CACHE cc) +(VALUE str, CALL_DATA cd) () (VALUE val) { val = vm_opt_str_freeze(str, BOP_FREEZE, idFreeze); - if (val == Qundef) { + if (UNDEF_P(val)) { PUSH(rb_str_resurrect(str)); CALL_SIMPLE_METHOD(); } } +/* optimized nil? */ +DEFINE_INSN +opt_nil_p +(CALL_DATA cd) +(VALUE recv) +(VALUE val) +// attr bool zjit_profile = true; +{ + val = vm_opt_nil_p(GET_CFP(), cd, recv); + + if (UNDEF_P(val)) { + CALL_SIMPLE_METHOD(); + } +} + DEFINE_INSN opt_str_uminus -(VALUE str, CALL_INFO ci, CALL_CACHE cc) +(VALUE str, CALL_DATA cd) () (VALUE val) { val = vm_opt_str_freeze(str, BOP_UMINUS, idUMinus); - if (val == Qundef) { + if (UNDEF_P(val)) { PUSH(rb_str_resurrect(str)); CALL_SIMPLE_METHOD(); } } DEFINE_INSN -opt_newarray_max -(rb_num_t num) +opt_duparray_send +(VALUE ary, ID method, rb_num_t argc) +(...) +(VALUE val) +/* This instruction typically has no funcalls. But it may compare array + * contents to each other which may call methods when necessary. + * No way to detect such method calls beforehand. + * We must mark it as not leaf. */ +// attr bool leaf = false; /* has rb_funcall() */ +// attr rb_snum_t sp_inc = 1 - (rb_snum_t)argc; +// attr rb_snum_t comptime_sp_inc = 1 - (rb_snum_t)argc; +{ + switch (method) { + case idIncludeP: + val = vm_opt_duparray_include_p(ec, ary, TOPN(0)); + break; + default: + rb_bug("unreachable"); + } +} + +DEFINE_INSN +opt_newarray_send +(rb_num_t num, rb_num_t method) (...) (VALUE val) /* This instruction typically has no funcalls. But it compares array @@ -833,34 +1063,89 @@ opt_newarray_max * cannot but mark it being not leaf. */ // attr bool leaf = false; /* has rb_funcall() */ // attr rb_snum_t sp_inc = 1 - (rb_snum_t)num; -{ - val = vm_opt_newarray_max(num, STACK_ADDR_FROM_TOP(num)); +// attr rb_snum_t comptime_sp_inc = 1 - (rb_snum_t)num; +{ + switch (method) { + case VM_OPT_NEWARRAY_SEND_HASH: + val = vm_opt_newarray_hash(ec, num, STACK_ADDR_FROM_TOP(num)); + break; + case VM_OPT_NEWARRAY_SEND_MIN: + val = vm_opt_newarray_min(ec, num, STACK_ADDR_FROM_TOP(num)); + break; + case VM_OPT_NEWARRAY_SEND_MAX: + val = vm_opt_newarray_max(ec, num, STACK_ADDR_FROM_TOP(num)); + break; + case VM_OPT_NEWARRAY_SEND_INCLUDE_P: + val = vm_opt_newarray_include_p(ec, (long)num-1, STACK_ADDR_FROM_TOP(num), TOPN(0)); + break; + case VM_OPT_NEWARRAY_SEND_PACK: + val = vm_opt_newarray_pack_buffer(ec, (long)num-1, STACK_ADDR_FROM_TOP(num), TOPN(0), Qundef); + break; + case VM_OPT_NEWARRAY_SEND_PACK_BUFFER: + val = vm_opt_newarray_pack_buffer(ec, (long)num-2, STACK_ADDR_FROM_TOP(num), TOPN(1), TOPN(0)); + break; + default: + rb_bug("unreachable"); + } } +/* super(args) # args.size => num */ DEFINE_INSN -opt_newarray_min -(rb_num_t num) +invokesuper +(CALL_DATA cd, ISEQ blockiseq) (...) (VALUE val) -/* Same discussion as opt_newarray_max. */ -// attr bool leaf = false; /* has rb_funcall() */ -// attr rb_snum_t sp_inc = 1 - (rb_snum_t)num; -{ - val = vm_opt_newarray_min(num, STACK_ADDR_FROM_TOP(num)); +// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); +// attr bool zjit_profile = true; +{ + struct rb_callinfo adjusted_ci = VM_CI_ON_STACK(vm_ci_mid(cd->ci), + vm_ci_flag(cd->ci), + vm_ci_argc(cd->ci), + vm_ci_kwarg(cd->ci)); + const struct rb_callcache *original_cc = rbimpl_atomic_ptr_load((void **)&cd->cc, RBIMPL_ATOMIC_ACQUIRE); + struct rb_call_data adjusted_cd = { + .ci = &adjusted_ci, + .cc = original_cc, + }; + + VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), adjusted_cd.ci, blockiseq, true); + val = vm_sendish(ec, GET_CFP(), &adjusted_cd, bh, mexp_search_super); + JIT_EXEC(ec, val); + + if (original_cc != adjusted_cd.cc && vm_cc_markable(adjusted_cd.cc)) { + rbimpl_atomic_ptr_store((volatile void **)&cd->cc, (void *)adjusted_cd.cc, RBIMPL_ATOMIC_RELEASE); + RB_OBJ_WRITTEN(GET_ISEQ(), Qundef, adjusted_cd.cc); + } + + if (UNDEF_P(val)) { + RESTORE_REGS(); + NEXT_INSN(); + } } /* super(args) # args.size => num */ DEFINE_INSN -invokesuper -(CALL_INFO ci, CALL_CACHE cc, ISEQ blockiseq) +invokesuperforward +(CALL_DATA cd, ISEQ blockiseq) (...) (VALUE val) -// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); +// attr rb_snum_t sp_inc = sp_inc_of_sendish(cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); { - VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), ci, blockiseq, true); - val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_super_method); + struct rb_forwarding_call_data adjusted_cd; + struct rb_callinfo adjusted_ci; + + VALUE bh = vm_caller_setup_fwd_args(ec, GET_CFP(), cd, blockiseq, 1, &adjusted_cd, &adjusted_ci); + + val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super); + JIT_EXEC(ec, val); + + if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { + RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + } - if (val == Qundef) { + if (UNDEF_P(val)) { RESTORE_REGS(); NEXT_INSN(); } @@ -869,20 +1154,19 @@ invokesuper /* yield(args) */ DEFINE_INSN invokeblock -(CALL_INFO ci) +(CALL_DATA cd) (...) (VALUE val) // attr bool handles_sp = true; -// attr rb_snum_t sp_inc = sp_inc_of_invokeblock(ci); +// attr rb_snum_t sp_inc = sp_inc_of_invokeblock(cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_invokeblock(ci); +// attr bool zjit_profile = true; { - static struct rb_call_cache cc = { - 0, 0, NULL, vm_invokeblock_i, - }; - VALUE bh = VM_BLOCK_HANDLER_NONE; - val = vm_sendish(ec, GET_CFP(), ci, &cc, bh, vm_search_invokeblock); + val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock); + JIT_EXEC(ec, val); - if (val == Qundef) { + if (UNDEF_P(val)) { RESTORE_REGS(); NEXT_INSN(); } @@ -904,21 +1188,19 @@ leave const VALUE *const bp = vm_base_ptr(GET_CFP()); if (GET_SP() != bp) { vm_stack_consistency_error(ec, GET_CFP(), bp); - } + } } - RUBY_VM_CHECK_INTS(ec); - if (vm_pop_frame(ec, GET_CFP(), GET_EP())) { #if OPT_CALL_THREADED_CODE - rb_ec_thread_ptr(ec)->retval = val; - return 0; + rb_ec_thread_ptr(ec)->retval = val; + return 0; #else - return val; + return val; #endif } else { - RESTORE_REGS(); + RESTORE_REGS(); } } @@ -935,7 +1217,6 @@ throw /* Same discussion as leave. */ // attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */ { - RUBY_VM_CHECK_INTS(ec); val = vm_throw(ec, GET_CFP(), throw_state, throwobj); THROW_EXCEPTION(val); /* unreachable */ @@ -968,8 +1249,8 @@ branchif // attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */ { if (RTEST(val)) { - RUBY_VM_CHECK_INTS(ec); - JUMP(dst); + RUBY_VM_CHECK_INTS(ec); + JUMP(dst); } } @@ -983,8 +1264,8 @@ branchunless // attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */ { if (!RTEST(val)) { - RUBY_VM_CHECK_INTS(ec); - JUMP(dst); + RUBY_VM_CHECK_INTS(ec); + JUMP(dst); } } @@ -998,41 +1279,65 @@ branchnil // attr bool leaf = false; /* has rb_threadptr_execute_interrupts() */ { if (NIL_P(val)) { - RUBY_VM_CHECK_INTS(ec); - JUMP(dst); + RUBY_VM_CHECK_INTS(ec); + JUMP(dst); } } -/**********************************************************/ -/* for optimize */ -/**********************************************************/ - -/* push inline-cached value and go to dst if it is valid */ +/* same as jump, but without interrupt check */ DEFINE_INSN -opt_getinlinecache -(OFFSET dst, IC ic) +jump_without_ints +(OFFSET dst) +() () +// attr bool leaf = true; +{ + JUMP(dst); +} + +/* same as branchif, but without interrupt check */ +DEFINE_INSN +branchif_without_ints +(OFFSET dst) (VALUE val) +() +// attr bool leaf = true; { - if (vm_ic_hit_p(ic, GET_EP())) { - val = ic->ic_value.value; - JUMP(dst); - } - else { - val = Qnil; + if (RTEST(val)) { + JUMP(dst); } } -/* set inline cache */ +/* same as branchunless, but without interrupt check */ DEFINE_INSN -opt_setinlinecache -(IC ic) +branchunless_without_ints +(OFFSET dst) (VALUE val) +() +// attr bool leaf = true; +{ + if (!RTEST(val)) { + JUMP(dst); + } +} + +/* same as branchnil, but without interrupt check */ +DEFINE_INSN +branchnil_without_ints +(OFFSET dst) (VALUE val) +() +// attr bool leaf = true; { - vm_ic_update(ic, val, GET_EP()); + if (NIL_P(val)) { + JUMP(dst); + } } +/**********************************************************/ +/* for optimize */ +/**********************************************************/ + /* run iseq only once */ DEFINE_INSN once @@ -1054,7 +1359,7 @@ opt_case_dispatch OFFSET dst = vm_case_dispatch(hash, else_offset, key); if (dst) { - JUMP(dst); + JUMP(dst); } } @@ -1063,16 +1368,14 @@ opt_case_dispatch /* optimized X+Y. */ DEFINE_INSN opt_plus -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) -/* Array + anything can be handled inside of opt_plus, and that - * anything is converted into array using #to_ary. */ -// attr bool leaf = false; /* has rb_to_array_type() */ +// attr bool zjit_profile = true; { val = vm_opt_plus(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1080,13 +1383,14 @@ opt_plus /* optimized X-Y. */ DEFINE_INSN opt_minus -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_minus(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1094,13 +1398,14 @@ opt_minus /* optimized X*Y. */ DEFINE_INSN opt_mult -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_mult(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1108,16 +1413,17 @@ opt_mult /* optimized X/Y. */ DEFINE_INSN opt_div -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) /* In case of division by zero, it raises. Thus * ZeroDivisionError#initialize is called. */ // attr bool leaf = false; +// attr bool zjit_profile = true; { val = vm_opt_div(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1125,15 +1431,16 @@ opt_div /* optimized X%Y. */ DEFINE_INSN opt_mod -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) -/* Same discussion as opt_mod. */ +/* Same discussion as opt_div. */ // attr bool leaf = false; +// attr bool zjit_profile = true; { val = vm_opt_mod(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1141,17 +1448,14 @@ opt_mod /* optimized X==Y. */ DEFINE_INSN opt_eq -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) -/* This instruction can compare a string with non-string. This - * (somewhat) coerces the non-string into a string, via a method - * call. */ -// attr bool leaf = false; /* has rb_str_equal() */ +// attr bool zjit_profile = true; { - val = opt_eq_func(recv, obj, ci, cc); + val = opt_equality(GET_CFP(), recv, obj, cd); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1159,15 +1463,14 @@ opt_eq /* optimized X!=Y. */ DEFINE_INSN opt_neq -(CALL_INFO ci_eq, CALL_CACHE cc_eq, CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd_eq, CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) -/* Same discussion as opt_eq. */ -// attr bool leaf = false; /* has rb_str_equal() */ +// attr bool zjit_profile = true; { - val = vm_opt_neq(ci, cc, ci_eq, cc_eq, recv, obj); + val = vm_opt_neq(GET_CFP(), cd, cd_eq, recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1175,13 +1478,14 @@ opt_neq /* optimized X<Y. */ DEFINE_INSN opt_lt -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_lt(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1189,13 +1493,14 @@ opt_lt /* optimized X<=Y. */ DEFINE_INSN opt_le -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_le(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1203,13 +1508,14 @@ opt_le /* optimized X>Y. */ DEFINE_INSN opt_gt -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_gt(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1217,13 +1523,14 @@ opt_gt /* optimized X>=Y. */ DEFINE_INSN opt_ge -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_ge(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1231,17 +1538,18 @@ opt_ge /* << */ DEFINE_INSN opt_ltlt -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) /* This instruction can append an integer, as a codepoint, into a * string. Then what happens if that codepoint does not exist in the * string's encoding? Of course an exception. That's not a leaf. */ // attr bool leaf = false; /* has "invalid codepoint" exception */ +// attr bool zjit_profile = true; { val = vm_opt_ltlt(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1249,13 +1557,14 @@ opt_ltlt /* optimized X&Y. */ DEFINE_INSN opt_and -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_and(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1263,13 +1572,14 @@ opt_and /* optimized X|Y. */ DEFINE_INSN opt_or -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_or(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1277,7 +1587,7 @@ opt_or /* [] */ DEFINE_INSN opt_aref -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) /* This is complicated. In case of hash, vm_opt_aref() resorts to @@ -1285,10 +1595,11 @@ opt_aref * default_proc. This is a method call. So opt_aref is * (surprisingly) not leaf. */ // attr bool leaf = false; /* has rb_funcall() */ /* calls #yield */ +// attr bool zjit_profile = true; { val = vm_opt_aref(recv, obj); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1296,58 +1607,17 @@ opt_aref /* recv[obj] = set */ DEFINE_INSN opt_aset -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj, VALUE set) (VALUE val) /* This is another story than opt_aref. When vm_opt_aset() resorts * to rb_hash_aset(), which should call #hash for `obj`. */ // attr bool leaf = false; /* has rb_funcall() */ /* calls #hash */ +// attr bool zjit_profile = true; { val = vm_opt_aset(recv, obj, set); - if (val == Qundef) { - CALL_SIMPLE_METHOD(); - } -} - -/* recv[str] = set */ -DEFINE_INSN -opt_aset_with -(VALUE key, CALL_INFO ci, CALL_CACHE cc) -(VALUE recv, VALUE val) -(VALUE val) -/* Same discussion as opt_aset. */ -// attr bool leaf = false; /* has rb_funcall() */ /* calls #hash */ -{ - VALUE tmp = vm_opt_aset_with(recv, key, val); - - if (tmp != Qundef) { - val = tmp; - } - else { -#ifndef MJIT_HEADER - TOPN(0) = rb_str_resurrect(key); - PUSH(val); -#endif - CALL_SIMPLE_METHOD(); - } -} - -/* recv[str] */ -DEFINE_INSN -opt_aref_with -(VALUE key, CALL_INFO ci, CALL_CACHE cc) -(VALUE recv) -(VALUE val) -/* Same discussion as opt_aref. */ -// attr bool leaf = false; /* has rb_funcall() */ /* calls #yield */ -{ - val = vm_opt_aref_with(recv, key); - - if (val == Qundef) { -#ifndef MJIT_HEADER - PUSH(rb_str_resurrect(key)); -#endif + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1355,13 +1625,14 @@ opt_aref_with /* optimized length */ DEFINE_INSN opt_length -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_length(recv, BOP_LENGTH); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1369,13 +1640,14 @@ opt_length /* optimized size */ DEFINE_INSN opt_size -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_length(recv, BOP_SIZE); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1383,13 +1655,14 @@ opt_size /* optimized empty? */ DEFINE_INSN opt_empty_p -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_empty_p(recv); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1397,13 +1670,14 @@ opt_empty_p /* optimized succ */ DEFINE_INSN opt_succ -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) +// attr bool zjit_profile = true; { val = vm_opt_succ(recv); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } @@ -1411,66 +1685,84 @@ opt_succ /* optimized not */ DEFINE_INSN opt_not -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) +// attr bool zjit_profile = true; { - val = vm_opt_not(ci, cc, recv); + val = vm_opt_not(GET_CFP(), cd, recv); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } -/* optimized regexp match */ -DEFINE_INSN -opt_regexpmatch1 -(VALUE recv) -(VALUE obj) -(VALUE val) -// attr bool leaf = false; /* rb_reg_prepare_enc() may rb_raise() that calls rb_funcallv() */ -{ - val = vm_opt_regexpmatch1(recv, obj); -} - /* optimized regexp match 2 */ DEFINE_INSN opt_regexpmatch2 -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE obj2, VALUE obj1) (VALUE val) // attr bool leaf = false; /* match_at() has rb_thread_check_ints() */ +// attr bool zjit_profile = true; { val = vm_opt_regexpmatch2(obj2, obj1); - if (val == Qundef) { + if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); } } -/* call native compiled method */ +/* call specific function with args */ DEFINE_INSN -opt_call_c_function -(rb_insn_func_t funcptr) +invokebuiltin +(RB_BUILTIN bf) +(...) +(VALUE val) +// attr bool leaf = false; /* anything can happen inside */ +// attr rb_snum_t sp_inc = 1 - bf->argc; +{ + val = vm_invoke_builtin(ec, reg_cfp, bf, STACK_ADDR_FROM_TOP(bf->argc)); +} + +/* call specific function with args (same parameters) */ +DEFINE_INSN +opt_invokebuiltin_delegate +(RB_BUILTIN bf, rb_num_t index) () +(VALUE val) +// attr bool leaf = false; /* anything can happen inside */ +{ + val = vm_invoke_builtin_delegate(ec, reg_cfp, bf, (unsigned int)index); +} + +/* call specific function with args (same parameters) and leave */ +DEFINE_INSN +opt_invokebuiltin_delegate_leave +(RB_BUILTIN bf, rb_num_t index) () +(VALUE val) // attr bool leaf = false; /* anything can happen inside */ -// attr bool handles_sp = true; { - reg_cfp = (funcptr)(ec, reg_cfp); + val = vm_invoke_builtin_delegate(ec, reg_cfp, bf, (unsigned int)index); - if (reg_cfp == 0) { - VALUE err = ec->errinfo; - ec->errinfo = Qnil; - THROW_EXCEPTION(err); + /* leave fastpath */ + /* TracePoint/return fallbacks this insn to opt_invokebuiltin_delegate */ + if (vm_pop_frame(ec, GET_CFP(), GET_EP())) { +#if OPT_CALL_THREADED_CODE + rb_ec_thread_ptr(ec)->retval = val; + return 0; +#else + return val; +#endif + } + else { + RESTORE_REGS(); } - - RESTORE_REGS(); - NEXT_INSN(); } /* BLT */ -DEFINE_INSN +DEFINE_INSN_IF(SUPPORT_JOKE) bitblt () () @@ -1480,7 +1772,7 @@ bitblt } /* The Answer to Life, the Universe, and Everything */ -DEFINE_INSN +DEFINE_INSN_IF(SUPPORT_JOKE) answer () () |