diff options
Diffstat (limited to 'compile.c')
| -rw-r--r-- | compile.c | 2740 |
1 files changed, 1894 insertions, 846 deletions
@@ -36,6 +36,7 @@ #include "internal/thread.h" #include "internal/variable.h" #include "iseq.h" +#include "ruby/ractor.h" #include "ruby/re.h" #include "ruby/util.h" #include "vm_core.h" @@ -47,11 +48,7 @@ #include "insns.inc" #include "insns_info.inc" -#undef RUBY_UNTYPED_DATA_WARNING -#define RUBY_UNTYPED_DATA_WARNING 0 - #define FIXNUM_INC(n, i) ((n)+(INT2FIX(i)&~FIXNUM_FLAG)) -#define FIXNUM_OR(n, i) ((n)|INT2FIX(i)) typedef struct iseq_link_element { enum { @@ -216,6 +213,9 @@ const ID rb_iseq_shared_exc_local_tbl[] = {idERROR_INFO}; #define NEW_CHILD_ISEQ(node, name, type, line_no) \ new_child_iseq(iseq, (node), rb_fstring(name), iseq, (type), (line_no)) +#define NEW_CHILD_ISEQ_WITH_CALLBACK(callback_func, name, type, line_no) \ + new_child_iseq_with_callback(iseq, (callback_func), (name), iseq, (type), (line_no)) + /* add instructions */ #define ADD_SEQ(seq1, seq2) \ APPEND_LIST((seq1), (seq2)) @@ -438,12 +438,10 @@ do { \ #define NO_CHECK(sub) (void)(sub) #define BEFORE_RETURN -/* leave name uninitialized so that compiler warn if INIT_ANCHOR is - * missing */ #define DECL_ANCHOR(name) \ - LINK_ANCHOR name[1] = {{{ISEQ_ELEMENT_ANCHOR,},}} + LINK_ANCHOR name[1] = {{{ISEQ_ELEMENT_ANCHOR,},&name[0].anchor}} #define INIT_ANCHOR(name) \ - (name->last = &name->anchor) + ((name->last = &name->anchor)->next = NULL) /* re-initialize */ static inline VALUE freeze_hide_obj(VALUE obj) @@ -489,15 +487,16 @@ static int iseq_setup_insn(rb_iseq_t *iseq, LINK_ANCHOR *const anchor); static int iseq_optimize(rb_iseq_t *iseq, LINK_ANCHOR *const anchor); static int iseq_insns_unification(rb_iseq_t *iseq, LINK_ANCHOR *const anchor); -static int iseq_set_local_table(rb_iseq_t *iseq, const rb_ast_id_table_t *tbl); +static int iseq_set_local_table(rb_iseq_t *iseq, const rb_ast_id_table_t *tbl, const NODE *const node_args); static int iseq_set_exception_local_table(rb_iseq_t *iseq); static int iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, const NODE *const node); static int iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor); static int iseq_set_exception_table(rb_iseq_t *iseq); static int iseq_set_optargs_table(rb_iseq_t *iseq); +static int iseq_set_parameters_lvar_state(const rb_iseq_t *iseq); -static int compile_defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, VALUE needstr); +static int compile_defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, VALUE needstr, bool ignore); static int compile_hash(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, int method_call_keywords, int popped); /* @@ -612,12 +611,25 @@ branch_coverage_valid_p(rb_iseq_t *iseq, int first_line) } static VALUE -decl_branch_base(rb_iseq_t *iseq, const NODE *node, const char *type) +setup_branch(const rb_code_location_t *loc, const char *type, VALUE structure, VALUE key) { - const int first_lineno = nd_first_lineno(node), first_column = nd_first_column(node); - const int last_lineno = nd_last_lineno(node), last_column = nd_last_column(node); + const int first_lineno = loc->beg_pos.lineno, first_column = loc->beg_pos.column; + const int last_lineno = loc->end_pos.lineno, last_column = loc->end_pos.column; + VALUE branch = rb_ary_hidden_new(6); + + rb_hash_aset(structure, key, branch); + rb_ary_push(branch, ID2SYM(rb_intern(type))); + rb_ary_push(branch, INT2FIX(first_lineno)); + rb_ary_push(branch, INT2FIX(first_column)); + rb_ary_push(branch, INT2FIX(last_lineno)); + rb_ary_push(branch, INT2FIX(last_column)); + return branch; +} - if (!branch_coverage_valid_p(iseq, first_lineno)) return Qundef; +static VALUE +decl_branch_base(rb_iseq_t *iseq, VALUE key, const rb_code_location_t *loc, const char *type) +{ + if (!branch_coverage_valid_p(iseq, loc->beg_pos.lineno)) return Qundef; /* * if !structure[node] @@ -628,18 +640,11 @@ decl_branch_base(rb_iseq_t *iseq, const NODE *node, const char *type) */ VALUE structure = RARRAY_AREF(ISEQ_BRANCH_COVERAGE(iseq), 0); - VALUE key = (VALUE)node | 1; // FIXNUM for hash key VALUE branch_base = rb_hash_aref(structure, key); VALUE branches; if (NIL_P(branch_base)) { - branch_base = rb_ary_hidden_new(6); - rb_hash_aset(structure, key, branch_base); - rb_ary_push(branch_base, ID2SYM(rb_intern(type))); - rb_ary_push(branch_base, INT2FIX(first_lineno)); - rb_ary_push(branch_base, INT2FIX(first_column)); - rb_ary_push(branch_base, INT2FIX(last_lineno)); - rb_ary_push(branch_base, INT2FIX(last_column)); + branch_base = setup_branch(loc, type, structure, key); branches = rb_hash_new(); rb_obj_hide(branches); rb_ary_push(branch_base, branches); @@ -661,12 +666,9 @@ generate_dummy_line_node(int lineno, int node_id) } static void -add_trace_branch_coverage(rb_iseq_t *iseq, LINK_ANCHOR *const seq, const NODE *node, int branch_id, const char *type, VALUE branches) +add_trace_branch_coverage(rb_iseq_t *iseq, LINK_ANCHOR *const seq, const rb_code_location_t *loc, int node_id, int branch_id, const char *type, VALUE branches) { - const int first_lineno = nd_first_lineno(node), first_column = nd_first_column(node); - const int last_lineno = nd_last_lineno(node), last_column = nd_last_column(node); - - if (!branch_coverage_valid_p(iseq, first_lineno)) return; + if (!branch_coverage_valid_p(iseq, loc->beg_pos.lineno)) return; /* * if !branches[branch_id] @@ -681,13 +683,7 @@ add_trace_branch_coverage(rb_iseq_t *iseq, LINK_ANCHOR *const seq, const NODE *n long counter_idx; if (NIL_P(branch)) { - branch = rb_ary_hidden_new(6); - rb_hash_aset(branches, key, branch); - rb_ary_push(branch, ID2SYM(rb_intern(type))); - rb_ary_push(branch, INT2FIX(first_lineno)); - rb_ary_push(branch, INT2FIX(first_column)); - rb_ary_push(branch, INT2FIX(last_lineno)); - rb_ary_push(branch, INT2FIX(last_column)); + branch = setup_branch(loc, type, branches, key); VALUE counters = RARRAY_AREF(ISEQ_BRANCH_COVERAGE(iseq), 1); counter_idx = RARRAY_LEN(counters); rb_ary_push(branch, LONG2FIX(counter_idx)); @@ -698,7 +694,7 @@ add_trace_branch_coverage(rb_iseq_t *iseq, LINK_ANCHOR *const seq, const NODE *n } ADD_TRACE_WITH_DATA(seq, RUBY_EVENT_COVERAGE_BRANCH, counter_idx); - ADD_SYNTHETIC_INSN(seq, last_lineno, nd_node_id(node), nop); + ADD_SYNTHETIC_INSN(seq, loc->end_pos.lineno, node_id, nop); } #define ISEQ_LAST_LINE(iseq) (ISEQ_COMPILE_DATA(iseq)->last_line) @@ -840,9 +836,9 @@ get_string_value(const NODE *node) { switch (nd_type(node)) { case NODE_STR: - return rb_node_str_string_val(node); + return RB_OBJ_SET_SHAREABLE(rb_node_str_string_val(node)); case NODE_FILE: - return rb_node_file_path_val(node); + return RB_OBJ_SET_SHAREABLE(rb_node_file_path_val(node)); default: rb_bug("unexpected node: %s", ruby_node_name(nd_type(node))); } @@ -862,25 +858,24 @@ rb_iseq_compile_callback(rb_iseq_t *iseq, const struct rb_iseq_new_with_callback return iseq_setup(iseq, ret); } +static bool drop_unreachable_return(LINK_ANCHOR *ret); + VALUE rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node) { DECL_ANCHOR(ret); INIT_ANCHOR(ret); - if (IMEMO_TYPE_P(node, imemo_ifunc)) { - rb_raise(rb_eArgError, "unexpected imemo_ifunc"); - } - if (node == 0) { NO_CHECK(COMPILE(ret, "nil", node)); - iseq_set_local_table(iseq, 0); + iseq_set_local_table(iseq, 0, 0); } /* assume node is T_NODE */ else if (nd_type_p(node, NODE_SCOPE)) { /* iseq type of top, method, class, block */ - iseq_set_local_table(iseq, RNODE_SCOPE(node)->nd_tbl); + iseq_set_local_table(iseq, RNODE_SCOPE(node)->nd_tbl, (NODE *)RNODE_SCOPE(node)->nd_args); iseq_set_arguments(iseq, ret, (NODE *)RNODE_SCOPE(node)->nd_args); + iseq_set_parameters_lvar_state(iseq); switch (ISEQ_BODY(iseq)->type) { case ISEQ_TYPE_BLOCK: @@ -965,7 +960,7 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node) ADD_GETLOCAL(ret, &dummy_line_node, LVAR_ERRINFO, 0); ADD_INSN1(ret, &dummy_line_node, throw, INT2FIX(0) /* continue throw */ ); } - else { + else if (!drop_unreachable_return(ret)) { ADD_SYNTHETIC_INSN(ret, ISEQ_COMPILE_DATA(iseq)->last_line, -1, leave); } @@ -999,6 +994,7 @@ rb_iseq_translate_threaded_code(rb_iseq_t *iseq) #if USE_YJIT rb_yjit_live_iseq_count++; + rb_yjit_iseq_alloc_count++; #endif return COMPILE_OK; @@ -1033,75 +1029,44 @@ rb_iseq_original_iseq(const rb_iseq_t *iseq) /* cold path */ /* definition of data structure for compiler */ /*********************************************/ -/* - * On 32-bit SPARC, GCC by default generates SPARC V7 code that may require - * 8-byte word alignment. On the other hand, Oracle Solaris Studio seems to - * generate SPARCV8PLUS code with unaligned memory access instructions. - * That is why the STRICT_ALIGNMENT is defined only with GCC. - */ -#if defined(__sparc) && SIZEOF_VOIDP == 4 && defined(__GNUC__) - #define STRICT_ALIGNMENT +#if defined(HAVE_TRUE_LONG_LONG) && SIZEOF_LONG_LONG > SIZEOF_VALUE +# define ALIGNMENT_SIZE SIZEOF_LONG_LONG +#else +# define ALIGNMENT_SIZE SIZEOF_VALUE #endif +#define PADDING_SIZE_MAX ((size_t)((ALIGNMENT_SIZE) - 1)) -/* - * Some OpenBSD platforms (including sparc64) require strict alignment. - */ -#if defined(__OpenBSD__) - #include <sys/endian.h> - #ifdef __STRICT_ALIGNMENT - #define STRICT_ALIGNMENT - #endif -#endif +#define ALIGNMENT_SIZE_OF(type) alignment_size_assert(RUBY_ALIGNOF(type), #type) -#ifdef STRICT_ALIGNMENT - #if defined(HAVE_TRUE_LONG_LONG) && SIZEOF_LONG_LONG > SIZEOF_VALUE - #define ALIGNMENT_SIZE SIZEOF_LONG_LONG - #else - #define ALIGNMENT_SIZE SIZEOF_VALUE - #endif - #define PADDING_SIZE_MAX ((size_t)((ALIGNMENT_SIZE) - 1)) - #define ALIGNMENT_SIZE_MASK PADDING_SIZE_MAX - /* Note: ALIGNMENT_SIZE == (2 ** N) is expected. */ -#else - #define PADDING_SIZE_MAX 0 -#endif /* STRICT_ALIGNMENT */ +static inline size_t +alignment_size_assert(size_t align, const char *type) +{ + RUBY_ASSERT((align & (align - 1)) == 0, + "ALIGNMENT_SIZE_OF(%s):%zd == (2 ** N) is expected", type, align); + return align; +} -#ifdef STRICT_ALIGNMENT /* calculate padding size for aligned memory access */ -static size_t -calc_padding(void *ptr, size_t size) +static inline size_t +calc_padding(void *ptr, size_t align) { size_t mis; size_t padding = 0; - mis = (size_t)ptr & ALIGNMENT_SIZE_MASK; + mis = (size_t)ptr & (align - 1); if (mis > 0) { - padding = ALIGNMENT_SIZE - mis; - } -/* - * On 32-bit sparc or equivalents, when a single VALUE is requested - * and padding == sizeof(VALUE), it is clear that no padding is needed. - */ -#if ALIGNMENT_SIZE > SIZEOF_VALUE - if (size == sizeof(VALUE) && padding == sizeof(VALUE)) { - padding = 0; + padding = align - mis; } -#endif return padding; } -#endif /* STRICT_ALIGNMENT */ static void * -compile_data_alloc_with_arena(struct iseq_compile_data_storage **arena, size_t size) +compile_data_alloc_with_arena(struct iseq_compile_data_storage **arena, size_t size, size_t align) { void *ptr = 0; struct iseq_compile_data_storage *storage = *arena; -#ifdef STRICT_ALIGNMENT - size_t padding = calc_padding((void *)&storage->buff[storage->pos], size); -#else - const size_t padding = 0; /* expected to be optimized by compiler */ -#endif /* STRICT_ALIGNMENT */ + size_t padding = calc_padding((void *)&storage->buff[storage->pos], align); if (size >= INT_MAX - padding) rb_memerror(); if (storage->pos + size + padding > storage->size) { @@ -1117,14 +1082,10 @@ compile_data_alloc_with_arena(struct iseq_compile_data_storage **arena, size_t s storage->next = 0; storage->pos = 0; storage->size = alloc_size; -#ifdef STRICT_ALIGNMENT - padding = calc_padding((void *)&storage->buff[storage->pos], size); -#endif /* STRICT_ALIGNMENT */ + padding = calc_padding((void *)&storage->buff[storage->pos], align); } -#ifdef STRICT_ALIGNMENT storage->pos += (int)padding; -#endif /* STRICT_ALIGNMENT */ ptr = (void *)&storage->buff[storage->pos]; storage->pos += (int)size; @@ -1132,51 +1093,60 @@ compile_data_alloc_with_arena(struct iseq_compile_data_storage **arena, size_t s } static void * -compile_data_alloc(rb_iseq_t *iseq, size_t size) +compile_data_alloc(rb_iseq_t *iseq, size_t size, size_t align) { struct iseq_compile_data_storage ** arena = &ISEQ_COMPILE_DATA(iseq)->node.storage_current; - return compile_data_alloc_with_arena(arena, size); + return compile_data_alloc_with_arena(arena, size, align); } +#define compile_data_alloc_type(iseq, type) \ + (type *)compile_data_alloc(iseq, sizeof(type), ALIGNMENT_SIZE_OF(type)) + static inline void * -compile_data_alloc2(rb_iseq_t *iseq, size_t x, size_t y) +compile_data_alloc2(rb_iseq_t *iseq, size_t elsize, size_t num, size_t align) { - size_t size = rb_size_mul_or_raise(x, y, rb_eRuntimeError); - return compile_data_alloc(iseq, size); + size_t size = rb_size_mul_or_raise(elsize, num, rb_eRuntimeError); + return compile_data_alloc(iseq, size, align); } +#define compile_data_alloc2_type(iseq, type, num) \ + (type *)compile_data_alloc2(iseq, sizeof(type), num, ALIGNMENT_SIZE_OF(type)) + static inline void * -compile_data_calloc2(rb_iseq_t *iseq, size_t x, size_t y) +compile_data_calloc2(rb_iseq_t *iseq, size_t elsize, size_t num, size_t align) { - size_t size = rb_size_mul_or_raise(x, y, rb_eRuntimeError); - void *p = compile_data_alloc(iseq, size); + size_t size = rb_size_mul_or_raise(elsize, num, rb_eRuntimeError); + void *p = compile_data_alloc(iseq, size, align); memset(p, 0, size); return p; } +#define compile_data_calloc2_type(iseq, type, num) \ + (type *)compile_data_calloc2(iseq, sizeof(type), num, ALIGNMENT_SIZE_OF(type)) + static INSN * compile_data_alloc_insn(rb_iseq_t *iseq) { struct iseq_compile_data_storage ** arena = &ISEQ_COMPILE_DATA(iseq)->insn.storage_current; - return (INSN *)compile_data_alloc_with_arena(arena, sizeof(INSN)); + return (INSN *)compile_data_alloc_with_arena(arena, sizeof(INSN), ALIGNMENT_SIZE_OF(INSN)); } static LABEL * compile_data_alloc_label(rb_iseq_t *iseq) { - return (LABEL *)compile_data_alloc(iseq, sizeof(LABEL)); + return compile_data_alloc_type(iseq, LABEL); } static ADJUST * compile_data_alloc_adjust(rb_iseq_t *iseq) { - return (ADJUST *)compile_data_alloc(iseq, sizeof(ADJUST)); + return compile_data_alloc_type(iseq, ADJUST); } static TRACE * compile_data_alloc_trace(rb_iseq_t *iseq) { - return (TRACE *)compile_data_alloc(iseq, sizeof(TRACE)); + return compile_data_alloc_type(iseq, TRACE); } /* @@ -1294,10 +1264,15 @@ static void APPEND_LIST(ISEQ_ARG_DECLARE LINK_ANCHOR *const anc1, LINK_ANCHOR *const anc2) { if (anc2->anchor.next) { + /* LINK_ANCHOR must not loop */ + RUBY_ASSERT(anc2->last != &anc2->anchor); anc1->last->next = anc2->anchor.next; anc2->anchor.next->prev = anc1->last; anc1->last = anc2->last; } + else { + RUBY_ASSERT(anc2->last == &anc2->anchor); + } verify_list("append", anc1); } #if CPDEBUG < 0 @@ -1357,6 +1332,7 @@ new_label_body(rb_iseq_t *iseq, long line) labelobj->set = 0; labelobj->rescued = LABEL_RESCUE_NONE; labelobj->unremovable = 0; + labelobj->position = -1; return labelobj; } @@ -1373,7 +1349,7 @@ new_adjust_body(rb_iseq_t *iseq, LABEL *label, int line) } static void -iseq_insn_each_markable_object(INSN *insn, void (*func)(VALUE, VALUE), VALUE data) +iseq_insn_each_markable_object(INSN *insn, void (*func)(VALUE *, VALUE), VALUE data) { const char *types = insn_op_types(insn->insn_id); for (int j = 0; types[j]; j++) { @@ -1384,7 +1360,7 @@ iseq_insn_each_markable_object(INSN *insn, void (*func)(VALUE, VALUE), VALUE dat case TS_VALUE: case TS_IC: // constant path array case TS_CALLDATA: // ci is stored. - func(OPERAND_AT(insn, j), data); + func(&OPERAND_AT(insn, j), data); break; default: break; @@ -1393,9 +1369,12 @@ iseq_insn_each_markable_object(INSN *insn, void (*func)(VALUE, VALUE), VALUE dat } static void -iseq_insn_each_object_write_barrier(VALUE obj, VALUE iseq) +iseq_insn_each_object_write_barrier(VALUE * obj, VALUE iseq) { - RB_OBJ_WRITTEN(iseq, Qundef, obj); + RB_OBJ_WRITTEN(iseq, Qundef, *obj); + RUBY_ASSERT(SPECIAL_CONST_P(*obj) || + RBASIC_CLASS(*obj) == 0 || // hidden + RB_OBJ_SHAREABLE_P(*obj)); } static INSN * @@ -1428,7 +1407,7 @@ new_insn_body(rb_iseq_t *iseq, int line_no, int node_id, enum ruby_vminsn_type i if (argc > 0) { int i; va_start(argv, argc); - operands = compile_data_alloc2(iseq, sizeof(VALUE), argc); + operands = compile_data_alloc2_type(iseq, VALUE, argc); for (i = 0; i < argc; i++) { VALUE v = va_arg(argv, VALUE); operands[i] = v; @@ -1438,21 +1417,45 @@ new_insn_body(rb_iseq_t *iseq, int line_no, int node_id, enum ruby_vminsn_type i return new_insn_core(iseq, line_no, node_id, insn_id, argc, operands); } +static INSN * +insn_replace_with_operands(rb_iseq_t *iseq, INSN *iobj, enum ruby_vminsn_type insn_id, int argc, ...) +{ + VALUE *operands = 0; + va_list argv; + if (argc > 0) { + int i; + va_start(argv, argc); + operands = compile_data_alloc2_type(iseq, VALUE, argc); + for (i = 0; i < argc; i++) { + VALUE v = va_arg(argv, VALUE); + operands[i] = v; + } + va_end(argv); + } + + iobj->insn_id = insn_id; + iobj->operand_size = argc; + iobj->operands = operands; + iseq_insn_each_markable_object(iobj, iseq_insn_each_object_write_barrier, (VALUE)iseq); + + return iobj; +} + static const struct rb_callinfo * new_callinfo(rb_iseq_t *iseq, ID mid, int argc, unsigned int flag, struct rb_callinfo_kwarg *kw_arg, int has_blockiseq) { VM_ASSERT(argc >= 0); - if (!(flag & (VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_BLOCKARG | VM_CALL_KW_SPLAT)) && - kw_arg == NULL && !has_blockiseq) { - flag |= VM_CALL_ARGS_SIMPLE; - } - if (kw_arg) { flag |= VM_CALL_KWARG; argc += kw_arg->keyword_len; } + if (!(flag & (VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_BLOCKARG | VM_CALL_KWARG | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING)) + && !has_blockiseq) { + flag |= VM_CALL_ARGS_SIMPLE; + } + ISEQ_BODY(iseq)->ci_size++; const struct rb_callinfo *ci = vm_ci_new(mid, flag, argc, kw_arg); RB_OBJ_WRITTEN(iseq, Qundef, ci); @@ -1462,14 +1465,23 @@ new_callinfo(rb_iseq_t *iseq, ID mid, int argc, unsigned int flag, struct rb_cal static INSN * new_insn_send(rb_iseq_t *iseq, int line_no, int node_id, ID id, VALUE argc, const rb_iseq_t *blockiseq, VALUE flag, struct rb_callinfo_kwarg *keywords) { - VALUE *operands = compile_data_calloc2(iseq, sizeof(VALUE), 2); + VALUE *operands = compile_data_calloc2_type(iseq, VALUE, 2); VALUE ci = (VALUE)new_callinfo(iseq, id, FIX2INT(argc), FIX2INT(flag), keywords, blockiseq != NULL); operands[0] = ci; operands[1] = (VALUE)blockiseq; if (blockiseq) { RB_OBJ_WRITTEN(iseq, Qundef, blockiseq); } - INSN *insn = new_insn_core(iseq, line_no, node_id, BIN(send), 2, operands); + + INSN *insn; + + if (vm_ci_flag((struct rb_callinfo *)ci) & VM_CALL_FORWARDING) { + insn = new_insn_core(iseq, line_no, node_id, BIN(sendforward), 2, operands); + } + else { + insn = new_insn_core(iseq, line_no, node_id, BIN(send), 2, operands); + } + RB_OBJ_WRITTEN(iseq, Qundef, ci); RB_GC_GUARD(ci); return insn; @@ -1480,20 +1492,16 @@ new_child_iseq(rb_iseq_t *iseq, const NODE *const node, VALUE name, const rb_iseq_t *parent, enum rb_iseq_type type, int line_no) { rb_iseq_t *ret_iseq; - rb_ast_body_t ast; - - ast.root = node; - ast.frozen_string_literal = -1; - ast.coverage_enabled = -1; - ast.script_lines = ISEQ_BODY(iseq)->variable.script_lines; + VALUE ast_value = rb_ruby_ast_new(node); debugs("[new_child_iseq]> ---------------------------------------\n"); int isolated_depth = ISEQ_COMPILE_DATA(iseq)->isolated_depth; - ret_iseq = rb_iseq_new_with_opt(&ast, name, + ret_iseq = rb_iseq_new_with_opt(ast_value, name, rb_iseq_path(iseq), rb_iseq_realpath(iseq), line_no, parent, isolated_depth ? isolated_depth + 1 : 0, - type, ISEQ_COMPILE_DATA(iseq)->option); + type, ISEQ_COMPILE_DATA(iseq)->option, + ISEQ_BODY(iseq)->variable.script_lines); debugs("[new_child_iseq]< ---------------------------------------\n"); return ret_iseq; } @@ -1658,7 +1666,7 @@ iseq_setup(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) debugs("[compile step 6.1 (remove unused catch tables)] \n"); RUBY_ASSERT(ISEQ_COMPILE_DATA(iseq)); if (!ISEQ_COMPILE_DATA(iseq)->catch_except_p && ISEQ_BODY(iseq)->catch_table) { - xfree(ISEQ_BODY(iseq)->catch_table); + ruby_xfree_sized(ISEQ_BODY(iseq)->catch_table, iseq_catch_table_bytes(ISEQ_BODY(iseq)->catch_table->size)); ISEQ_BODY(iseq)->catch_table = NULL; } @@ -1684,6 +1692,7 @@ iseq_set_exception_local_table(rb_iseq_t *iseq) { ISEQ_BODY(iseq)->local_table_size = numberof(rb_iseq_shared_exc_local_tbl); ISEQ_BODY(iseq)->local_table = rb_iseq_shared_exc_local_tbl; + ISEQ_BODY(iseq)->lvar_states = NULL; // $! is read-only, so don't need lvar_states return COMPILE_OK; } @@ -1831,6 +1840,46 @@ iseq_lvar_id(const rb_iseq_t *iseq, int idx, int level) } static void +update_lvar_state(const rb_iseq_t *iseq, int level, int idx) +{ + for (int i=0; i<level; i++) { + iseq = ISEQ_BODY(iseq)->parent_iseq; + } + + enum lvar_state *states = ISEQ_BODY(iseq)->lvar_states; + int table_idx = ISEQ_BODY(iseq)->local_table_size - idx; + switch (states[table_idx]) { + case lvar_uninitialized: + states[table_idx] = lvar_initialized; + break; + case lvar_initialized: + states[table_idx] = lvar_reassigned; + break; + case lvar_reassigned: + /* nothing */ + break; + default: + rb_bug("unreachable"); + } +} + +static int +iseq_set_parameters_lvar_state(const rb_iseq_t *iseq) +{ + for (unsigned int i=0; i<ISEQ_BODY(iseq)->param.size; i++) { + ISEQ_BODY(iseq)->lvar_states[i] = lvar_initialized; + } + + int lead_num = ISEQ_BODY(iseq)->param.lead_num; + int opt_num = ISEQ_BODY(iseq)->param.opt_num; + for (int i=0; i<opt_num; i++) { + ISEQ_BODY(iseq)->lvar_states[lead_num + i] = lvar_uninitialized; + } + + return COMPILE_OK; +} + +static void iseq_add_getlocal(rb_iseq_t *iseq, LINK_ANCHOR *const seq, const NODE *const line_node, int idx, int level) { if (iseq_local_block_param_p(iseq, idx, level)) { @@ -1851,6 +1900,7 @@ iseq_add_setlocal(rb_iseq_t *iseq, LINK_ANCHOR *const seq, const NODE *const lin else { ADD_INSN2(seq, line_node, setlocal, INT2FIX((idx) + VM_ENV_DATA_SIZE - 1), INT2FIX(level)); } + update_lvar_state(iseq, level, idx); if (level > 0) access_outer_variables(iseq, level, iseq_lvar_id(iseq, idx, level), Qtrue); } @@ -1925,9 +1975,6 @@ iseq_set_arguments_keywords(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, } else { switch (nd_type(val_node)) { - case NODE_LIT: - dv = RNODE_LIT(val_node)->nd_lit; - break; case NODE_SYM: dv = rb_node_sym_string_val(val_node); break; @@ -1976,7 +2023,7 @@ iseq_set_arguments_keywords(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, keyword->num = kw; if (RNODE_DVAR(args->kw_rest_arg)->nd_vid != 0) { - ID kw_id = iseq->body->local_table[arg_size]; + ID kw_id = ISEQ_BODY(iseq)->local_table[arg_size]; keyword->rest_start = arg_size++; body->param.flags.has_kwrest = TRUE; @@ -1985,16 +2032,14 @@ iseq_set_arguments_keywords(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, keyword->required_num = rkw; keyword->table = &body->local_table[keyword->bits_start - keyword->num]; - { + if (RARRAY_LEN(default_values)) { VALUE *dvs = ALLOC_N(VALUE, RARRAY_LEN(default_values)); for (i = 0; i < RARRAY_LEN(default_values); i++) { VALUE dv = RARRAY_AREF(default_values, i); if (dv == complex_mark) dv = Qundef; - if (!SPECIAL_CONST_P(dv)) { - RB_OBJ_WRITTEN(iseq, Qundef, dv); - } - dvs[i] = dv; + if (!SPECIAL_CONST_P(dv)) rb_ractor_make_shareable(dv); + RB_OBJ_WRITE(iseq, &dvs[i], dv); } keyword->default_values = dvs; @@ -2002,6 +2047,22 @@ iseq_set_arguments_keywords(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, return arg_size; } +static void +iseq_set_use_block(rb_iseq_t *iseq) +{ + struct rb_iseq_constant_body *const body = ISEQ_BODY(iseq); + if (!body->param.flags.use_block) { + body->param.flags.use_block = 1; + + rb_vm_t *vm = GET_VM(); + + if (!rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK)) { + st_data_t key = (st_data_t)rb_intern_str(body->location.label); // String -> ID + set_insert(&vm->unused_block_warning_table, key); + } + } +} + static int iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *const node_args) { @@ -2009,7 +2070,7 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons if (node_args) { struct rb_iseq_constant_body *const body = ISEQ_BODY(iseq); - struct rb_args_info *args = &RNODE_ARGS(node_args)->nd_ainfo; + const struct rb_args_info *const args = &RNODE_ARGS(node_args)->nd_ainfo; ID rest_id = 0; int last_comma = 0; ID block_id = 0; @@ -2017,7 +2078,6 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons EXPECT_NODE("iseq_set_arguments", node_args, NODE_ARGS, COMPILE_NG); - body->param.flags.ruby2_keywords = args->ruby2_keywords; body->param.lead_num = arg_size = (int)args->pre_args_num; if (body->param.lead_num > 0) body->param.flags.has_lead = TRUE; debugs(" - argc: %d\n", body->param.lead_num); @@ -2029,6 +2089,13 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons } block_id = args->block_arg; + bool optimized_forward = (args->forwarding && args->pre_args_num == 0 && !args->opt_args); + + if (optimized_forward) { + rest_id = 0; + block_id = 0; + } + if (args->opt_args) { const rb_node_opt_arg_t *node = args->opt_args; LABEL *label; @@ -2085,8 +2152,8 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons if (args->kw_args) { arg_size = iseq_set_arguments_keywords(iseq, optargs, args, arg_size); } - else if (args->kw_rest_arg) { - ID kw_id = iseq->body->local_table[arg_size]; + else if (args->kw_rest_arg && !optimized_forward) { + ID kw_id = ISEQ_BODY(iseq)->local_table[arg_size]; struct rb_iseq_param_keyword *keyword = ZALLOC_N(struct rb_iseq_param_keyword, 1); keyword->rest_start = arg_size++; body->param.keyword = keyword; @@ -2100,9 +2167,20 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons body->param.flags.accepts_no_kwarg = TRUE; } - if (block_id) { + if (args->no_blockarg) { + body->param.flags.accepts_no_block = TRUE; + } + else if (block_id) { body->param.block_start = arg_size++; body->param.flags.has_block = TRUE; + iseq_set_use_block(iseq); + } + + // Only optimize specifically methods like this: `foo(...)` + if (optimized_forward) { + body->param.flags.use_block = 1; + body->param.flags.forwardable = TRUE; + arg_size = 1; } iseq_calc_param_size(iseq); @@ -2134,14 +2212,36 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons } static int -iseq_set_local_table(rb_iseq_t *iseq, const rb_ast_id_table_t *tbl) +iseq_set_local_table(rb_iseq_t *iseq, const rb_ast_id_table_t *tbl, const NODE *const node_args) { unsigned int size = tbl ? tbl->size : 0; + unsigned int offset = 0; + + if (node_args) { + struct rb_args_info *args = &RNODE_ARGS(node_args)->nd_ainfo; + + // If we have a function that only has `...` as the parameter, + // then its local table should only be `...` + // FIXME: I think this should be fixed in the AST rather than special case here. + if (args->forwarding && args->pre_args_num == 0 && !args->opt_args) { + CHECK(size >= 3); + size -= 3; + offset += 3; + } + } if (size > 0) { - ID *ids = (ID *)ALLOC_N(ID, size); - MEMCPY(ids, tbl->ids, ID, size); + ID *ids = ALLOC_N(ID, size); + MEMCPY(ids, tbl->ids + offset, ID, size); ISEQ_BODY(iseq)->local_table = ids; + + enum lvar_state *states = ALLOC_N(enum lvar_state, size); + // fprintf(stderr, "iseq:%p states:%p size:%d\n", iseq, states, (int)size); + for (unsigned int i=0; i<size; i++) { + states[i] = lvar_uninitialized; + // fprintf(stderr, "id:%s\n", rb_id2name(ISEQ_BODY(iseq)->local_table[i])); + } + ISEQ_BODY(iseq)->lvar_states = states; } ISEQ_BODY(iseq)->local_table_size = size; @@ -2230,6 +2330,33 @@ static const struct st_hash_type cdhash_type = { rb_iseq_cdhash_hash, }; +static VALUE +cdhash_new(size_t size) +{ + VALUE cdhash = rb_imemo_cdhash_new(size, &cdhash_type); + RB_OBJ_SET_SHAREABLE(cdhash); + return cdhash; +} + +static void +cdhash_aset(VALUE cdhash, VALUE key, VALUE val) +{ + st_table *tbl = rb_imemo_cdhash_tbl(cdhash); + st_insert(tbl, key, val); + RB_OBJ_WRITTEN(cdhash, Qundef, key); +} + +static void +cdhash_aset_if_missing(VALUE cdhash, VALUE key, VALUE val) +{ + st_table *tbl = rb_imemo_cdhash_tbl(cdhash); + VALUE dontcare; + if (!st_lookup(tbl, key, &dontcare)) { + st_insert(tbl, key, val); + RB_OBJ_WRITTEN(cdhash, Qundef, key); + } +} + struct cdhash_set_label_struct { VALUE hash; int pos; @@ -2237,15 +2364,20 @@ struct cdhash_set_label_struct { }; static int -cdhash_set_label_i(VALUE key, VALUE val, VALUE ptr) +cdhash_set_label_check_i(st_data_t key, st_data_t value, st_data_t argp, int error) +{ + return ST_REPLACE; +} + +static int +cdhash_set_label_replace_i(st_data_t *key, st_data_t *value, st_data_t ptr, int existing) { struct cdhash_set_label_struct *data = (struct cdhash_set_label_struct *)ptr; - LABEL *lobj = (LABEL *)(val & ~1); - rb_hash_aset(data->hash, key, INT2FIX(lobj->position - (data->pos+data->len))); + LABEL *lobj = (LABEL *)(*value & ~1); + *value = lobj->position - (data->pos+data->len); return ST_CONTINUE; } - static inline VALUE get_ivar_ic_value(rb_iseq_t *iseq,ID id) { @@ -2275,8 +2407,8 @@ get_cvar_ic_value(rb_iseq_t *iseq,ID id) dump_disasm_list_with_cursor(FIRST_ELEMENT(anchor), list, dest) #define BADINSN_ERROR \ - (xfree(generated_iseq), \ - xfree(insns_info), \ + (SIZED_FREE_N(generated_iseq, generated_iseq_size), \ + SIZED_FREE_N(insns_info, insns_info_size), \ BADINSN_DUMP(anchor, list, NULL), \ COMPILE_ERROR) @@ -2424,6 +2556,7 @@ add_adjust_info(struct iseq_insn_info_entry *insns_info, unsigned int *positions int insns_info_index, int code_index, const ADJUST *adjust) { insns_info[insns_info_index].line_no = adjust->line_no; + insns_info[insns_info_index].node_id = -1; insns_info[insns_info_index].events = 0; positions[insns_info_index] = code_index; return TRUE; @@ -2435,7 +2568,7 @@ array_to_idlist(VALUE arr) RUBY_ASSERT(RB_TYPE_P(arr, T_ARRAY)); long size = RARRAY_LEN(arr); ID *ids = (ID *)ALLOC_N(ID, size + 1); - for (int i = 0; i < size; i++) { + for (long i = 0; i < size; i++) { VALUE sym = RARRAY_AREF(arr, i); ids[i] = SYM2ID(sym); } @@ -2541,8 +2674,13 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) } /* make instruction sequence */ + const int generated_iseq_size = code_index; generated_iseq = ALLOC_N(VALUE, code_index); + + const int insns_info_size = insn_num; insns_info = ALLOC_N(struct iseq_insn_info_entry, insn_num); + + const int positions_size = insn_num; positions = ALLOC_N(unsigned int, insn_num); if (ISEQ_IS_SIZE(body)) { body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, ISEQ_IS_SIZE(body)); @@ -2550,7 +2688,13 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) else { body->is_entries = NULL; } - body->call_data = ZALLOC_N(struct rb_call_data, body->ci_size); + + if (body->ci_size) { + body->call_data = ZALLOC_N(struct rb_call_data, body->ci_size); + } + else { + body->call_data = NULL; + } ISEQ_COMPILE_DATA(iseq)->ci_index = 0; // Calculate the bitmask buffer size. @@ -2561,16 +2705,22 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) iseq_bits_t * mark_offset_bits; int code_size = code_index; - iseq_bits_t tmp[1] = {0}; bool needs_bitmap = false; - if (ISEQ_MBITS_BUFLEN(code_index) == 1) { - mark_offset_bits = tmp; + const size_t mark_offset_bits_size = ISEQ_MBITS_BUFLEN(code_index); + if (mark_offset_bits_size == 1) { + mark_offset_bits = &ISEQ_COMPILE_DATA(iseq)->mark_bits.single; + ISEQ_COMPILE_DATA(iseq)->is_single_mark_bit = true; } else { - mark_offset_bits = ZALLOC_N(iseq_bits_t, ISEQ_MBITS_BUFLEN(code_index)); + mark_offset_bits = ZALLOC_N(iseq_bits_t, mark_offset_bits_size); + ISEQ_COMPILE_DATA(iseq)->mark_bits.list = mark_offset_bits; + ISEQ_COMPILE_DATA(iseq)->is_single_mark_bit = false; } + ISEQ_COMPILE_DATA(iseq)->iseq_encoded = (void *)generated_iseq; + ISEQ_COMPILE_DATA(iseq)->iseq_size = code_index; + list = FIRST_ELEMENT(anchor); insns_info_index = code_index = sp = 0; @@ -2611,10 +2761,8 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) data.hash = map; data.pos = code_index; data.len = len; - rb_hash_foreach(map, cdhash_set_label_i, (VALUE)&data); + st_foreach_with_replace(rb_imemo_cdhash_tbl(map), cdhash_set_label_check_i, cdhash_set_label_replace_i, (VALUE)&data); - rb_hash_rehash(map); - freeze_hide_obj(map); generated_iseq[code_index + 1 + j] = map; ISEQ_MBITS_SET(mark_offset_bits, code_index + 1 + j); RB_OBJ_WRITTEN(iseq, Qundef, map); @@ -2663,12 +2811,12 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) if (insn == BIN(setinstancevariable)) { cache->iv_set_name = SYM2ID(operands[j - 1]); + cache->value = IVAR_CACHE_INIT; } else { cache->iv_set_name = 0; + cache->value = rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET); } - - vm_ic_attr_index_initialize(cache, INVALID_SHAPE_ID); } case TS_ISE: /* inline storage entry: `once` insn */ case TS_ICVARC: /* inline cvar cache */ @@ -2755,11 +2903,11 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) } else if (diff < 0) { int label_no = adjust->label ? adjust->label->label_no : -1; - xfree(generated_iseq); - xfree(insns_info); - xfree(positions); + SIZED_FREE_N(generated_iseq, generated_iseq_size); + SIZED_FREE_N(insns_info, insns_info_size); + SIZED_FREE_N(positions, positions_size); if (ISEQ_MBITS_BUFLEN(code_size) > 1) { - xfree(mark_offset_bits); + SIZED_FREE_N(mark_offset_bits, ISEQ_MBITS_BUFLEN(code_index)); } debug_list(anchor, list); COMPILE_ERROR(iseq, adjust->line_no, @@ -2781,16 +2929,17 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) body->iseq_size = code_index; body->stack_max = stack_max; - if (ISEQ_MBITS_BUFLEN(body->iseq_size) == 1) { - body->mark_bits.single = mark_offset_bits[0]; + if (ISEQ_COMPILE_DATA(iseq)->is_single_mark_bit) { + body->mark_bits.single = ISEQ_COMPILE_DATA(iseq)->mark_bits.single; } else { if (needs_bitmap) { body->mark_bits.list = mark_offset_bits; } else { - body->mark_bits.list = 0; - ruby_xfree(mark_offset_bits); + body->mark_bits.list = NULL; + ISEQ_COMPILE_DATA(iseq)->mark_bits.list = NULL; + SIZED_FREE_N(mark_offset_bits, mark_offset_bits_size); } } @@ -2798,9 +2947,9 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) body->insns_info.body = insns_info; body->insns_info.positions = positions; - REALLOC_N(insns_info, struct iseq_insn_info_entry, insns_info_index); + SIZED_REALLOC_N(insns_info, struct iseq_insn_info_entry, insns_info_index, insns_info_size); body->insns_info.body = insns_info; - REALLOC_N(positions, unsigned int, insns_info_index); + SIZED_REALLOC_N(positions, unsigned int, insns_info_index, positions_size); body->insns_info.positions = positions; body->insns_info.size = insns_info_index; @@ -2838,11 +2987,16 @@ iseq_set_exception_table(rb_iseq_t *iseq) table->size = tlen; for (i = 0; i < table->size; i++) { + int pos; ptr = RARRAY_CONST_PTR(tptr[i]); entry = UNALIGNED_MEMBER_PTR(table, entries[i]); entry->type = (enum rb_catch_type)(ptr[0] & 0xffff); - entry->start = label_get_position((LABEL *)(ptr[1] & ~1)); - entry->end = label_get_position((LABEL *)(ptr[2] & ~1)); + pos = label_get_position((LABEL *)(ptr[1] & ~1)); + RUBY_ASSERT(pos >= 0); + entry->start = (unsigned int)pos; + pos = label_get_position((LABEL *)(ptr[2] & ~1)); + RUBY_ASSERT(pos >= 0); + entry->end = (unsigned int)pos; entry->iseq = (rb_iseq_t *)ptr[3]; RB_OBJ_WRITTEN(iseq, Qundef, entry->iseq); @@ -2966,16 +3120,18 @@ unref_destination(INSN *iobj, int pos) if (!lobj->refcnt) ELEM_REMOVE(&lobj->link); } -static void +static bool replace_destination(INSN *dobj, INSN *nobj) { VALUE n = OPERAND_AT(nobj, 0); LABEL *dl = (LABEL *)OPERAND_AT(dobj, 0); LABEL *nl = (LABEL *)n; + if (dl == nl) return false; --dl->refcnt; ++nl->refcnt; OPERAND_AT(dobj, 0) = n; if (!dl->refcnt) ELEM_REMOVE(&dl->link); + return true; } static LABEL* @@ -3008,7 +3164,6 @@ remove_unreachable_chunk(rb_iseq_t *iseq, LINK_ELEMENT *i) break; } else if ((lab = find_destination((INSN *)i)) != 0) { - if (lab->unremovable) break; unref_counts[lab->label_no]++; } } @@ -3025,8 +3180,7 @@ remove_unreachable_chunk(rb_iseq_t *iseq, LINK_ELEMENT *i) /* do nothing */ } else if (IS_ADJUST(i)) { - LABEL *dest = ((ADJUST *)i)->label; - if (dest && dest->unremovable) return 0; + return 0; } end = i; } while ((i = i->next) != 0); @@ -3071,7 +3225,7 @@ iseq_pop_newarray(rb_iseq_t *iseq, INSN *iobj) static int is_frozen_putstring(INSN *insn, VALUE *op) { - if (IS_INSN_ID(insn, putstring)) { + if (IS_INSN_ID(insn, dupstring) || IS_INSN_ID(insn, dupchilledstring)) { *op = OPERAND_AT(insn, 0); return 1; } @@ -3083,6 +3237,25 @@ is_frozen_putstring(INSN *insn, VALUE *op) } static int +insn_has_label_before(LINK_ELEMENT *elem) +{ + LINK_ELEMENT *prev = elem->prev; + while (prev) { + if (prev->type == ISEQ_ELEMENT_LABEL) { + LABEL *label = (LABEL *)prev; + if (label->refcnt > 0) { + return 1; + } + } + else if (prev->type == ISEQ_ELEMENT_INSN) { + break; + } + prev = prev->prev; + } + return 0; +} + +static int optimize_checktype(rb_iseq_t *iseq, INSN *iobj) { /* @@ -3112,7 +3285,8 @@ optimize_checktype(rb_iseq_t *iseq, INSN *iobj) VALUE type; switch (INSN_OF(iobj)) { - case BIN(putstring): + case BIN(dupstring): + case BIN(dupchilledstring): type = INT2FIX(T_STRING); break; case BIN(putnil): @@ -3190,33 +3364,7 @@ ci_argc_set(const rb_iseq_t *iseq, const struct rb_callinfo *ci, int argc) return nci; } -static bool -optimize_args_splat_no_copy(rb_iseq_t *iseq, INSN *insn, LINK_ELEMENT *niobj, - unsigned int set_flags, unsigned int unset_flags, unsigned int remove_flags) -{ - LINK_ELEMENT *iobj = (LINK_ELEMENT *)insn; - if ((set_flags & VM_CALL_ARGS_BLOCKARG) && (set_flags & VM_CALL_KW_SPLAT) && - IS_NEXT_INSN_ID(niobj, splatkw)) { - niobj = niobj->next; - } - if (!IS_NEXT_INSN_ID(niobj, send) && !IS_NEXT_INSN_ID(niobj, invokesuper)) { - return false; - } - niobj = niobj->next; - - const struct rb_callinfo *ci = (const struct rb_callinfo *)OPERAND_AT(niobj, 0); - unsigned int flags = vm_ci_flag(ci); - if ((flags & set_flags) == set_flags && !(flags & unset_flags)) { - RUBY_ASSERT(flags & VM_CALL_ARGS_SPLAT_MUT); - OPERAND_AT(iobj, 0) = Qfalse; - const struct rb_callinfo *nci = vm_ci_new(vm_ci_mid(ci), - flags & ~(VM_CALL_ARGS_SPLAT_MUT|remove_flags), vm_ci_argc(ci), vm_ci_kwarg(ci)); - RB_OBJ_WRITTEN(iseq, ci, nci); - OPERAND_AT(niobj, 0) = (VALUE)nci; - return true; - } - return false; -} +#define vm_ci_simple(ci) (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) static int iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcallopt) @@ -3256,9 +3404,10 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal * => in this case, first jump instruction should jump to * LABEL2 directly */ - replace_destination(iobj, diobj); - remove_unreachable_chunk(iseq, iobj->link.next); - goto again; + if (replace_destination(iobj, diobj)) { + remove_unreachable_chunk(iseq, iobj->link.next); + goto again; + } } else if (IS_INSN_ID(diobj, leave)) { /* @@ -3303,8 +3452,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal */ piobj->insn_id = (IS_INSN_ID(piobj, branchif)) ? BIN(branchunless) : BIN(branchif); - replace_destination(piobj, iobj); - if (refcnt <= 1) { + if (replace_destination(piobj, iobj) && refcnt <= 1) { ELEM_REMOVE(&iobj->link); } else { @@ -3336,8 +3484,8 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal } /* - * putstring "beg" - * putstring "end" + * dupstring "beg" + * dupstring "end" * newrange excl * * ==> @@ -3352,9 +3500,10 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal if ((end = (INSN *)get_prev_insn(range)) != 0 && is_frozen_putstring(end, &str_end) && (beg = (INSN *)get_prev_insn(end)) != 0 && - is_frozen_putstring(beg, &str_beg)) { + is_frozen_putstring(beg, &str_beg) && + !(insn_has_label_before(&beg->link) || insn_has_label_before(&end->link))) { int excl = FIX2INT(OPERAND_AT(range, 0)); - VALUE lit_range = rb_range_new(str_beg, str_end, excl); + VALUE lit_range = RB_OBJ_SET_SHAREABLE(rb_range_new(str_beg, str_end, excl)); ELEM_REMOVE(&beg->link); ELEM_REMOVE(&end->link); @@ -3384,6 +3533,89 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal } } + /* + * duparray [...] + * send <calldata!mid:freeze, argc:0, ARGS_SIMPLE>, nil + * => + * opt_ary_freeze [...], <calldata!mid:freeze, argc:0, ARGS_SIMPLE> + */ + if (IS_INSN_ID(iobj, duparray)) { + LINK_ELEMENT *next = iobj->link.next; + if (IS_INSN(next) && (IS_INSN_ID(next, send))) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(next, 0); + const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(next, 1); + + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && blockiseq == NULL && vm_ci_mid(ci) == idFreeze) { + VALUE ary = iobj->operands[0]; + rb_obj_reveal(ary, rb_cArray); + + insn_replace_with_operands(iseq, iobj, BIN(opt_ary_freeze), 2, ary, (VALUE)ci); + ELEM_REMOVE(next); + } + } + } + + /* + * duphash {...} + * send <calldata!mid:freeze, argc:0, ARGS_SIMPLE>, nil + * => + * opt_hash_freeze {...}, <calldata!mid:freeze, argc:0, ARGS_SIMPLE> + */ + if (IS_INSN_ID(iobj, duphash)) { + LINK_ELEMENT *next = iobj->link.next; + if (IS_INSN(next) && (IS_INSN_ID(next, send))) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(next, 0); + const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(next, 1); + + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && blockiseq == NULL && vm_ci_mid(ci) == idFreeze) { + VALUE hash = iobj->operands[0]; + rb_obj_reveal(hash, rb_cHash); + RB_OBJ_SET_SHAREABLE(hash); + + insn_replace_with_operands(iseq, iobj, BIN(opt_hash_freeze), 2, hash, (VALUE)ci); + ELEM_REMOVE(next); + } + } + } + + /* + * newarray 0 + * send <calldata!mid:freeze, argc:0, ARGS_SIMPLE>, nil + * => + * opt_ary_freeze [], <calldata!mid:freeze, argc:0, ARGS_SIMPLE> + */ + if (IS_INSN_ID(iobj, newarray) && iobj->operands[0] == INT2FIX(0)) { + LINK_ELEMENT *next = iobj->link.next; + if (IS_INSN(next) && (IS_INSN_ID(next, send))) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(next, 0); + const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(next, 1); + + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && blockiseq == NULL && vm_ci_mid(ci) == idFreeze) { + insn_replace_with_operands(iseq, iobj, BIN(opt_ary_freeze), 2, rb_cArray_empty_frozen, (VALUE)ci); + ELEM_REMOVE(next); + } + } + } + + /* + * newhash 0 + * send <calldata!mid:freeze, argc:0, ARGS_SIMPLE>, nil + * => + * opt_hash_freeze {}, <calldata!mid:freeze, argc:0, ARGS_SIMPLE> + */ + if (IS_INSN_ID(iobj, newhash) && iobj->operands[0] == INT2FIX(0)) { + LINK_ELEMENT *next = iobj->link.next; + if (IS_INSN(next) && (IS_INSN_ID(next, send))) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(next, 0); + const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(next, 1); + + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && blockiseq == NULL && vm_ci_mid(ci) == idFreeze) { + insn_replace_with_operands(iseq, iobj, BIN(opt_hash_freeze), 2, rb_cHash_empty_frozen, (VALUE)ci); + ELEM_REMOVE(next); + } + } + } + if (IS_INSN_ID(iobj, branchif) || IS_INSN_ID(iobj, branchnil) || IS_INSN_ID(iobj, branchunless)) { @@ -3433,7 +3665,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal for (;;) { if (IS_INSN(&nobj->link) && IS_INSN_ID(nobj, jump)) { - replace_destination(iobj, nobj); + if (!replace_destination(iobj, nobj)) break; } else if (prev_dup && IS_INSN_ID(nobj, dup) && !!(nobj = (INSN *)nobj->link.next) && @@ -3454,7 +3686,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal * dup * if L2 */ - replace_destination(iobj, nobj); + if (!replace_destination(iobj, nobj)) break; } else if (pobj) { /* @@ -3468,16 +3700,16 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal * => * jump L1 * - * putstring ".." + * dupstring ".." * if L1 * => * jump L1 * - * putstring ".." + * dupstring ".." * dup * if L1 * => - * putstring ".." + * dupstring ".." * jump L1 * */ @@ -3492,7 +3724,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal OPERAND_AT(pobj, 0) == Qfalse : FALSE); } - else if (IS_INSN_ID(pobj, putstring) || + else if (IS_INSN_ID(pobj, dupstring) || IS_INSN_ID(pobj, duparray) || IS_INSN_ID(pobj, newarray)) { cond = IS_INSN_ID(iobj, branchif); @@ -3530,7 +3762,7 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal if (IS_INSN_ID(iobj, pop)) { /* - * putself / putnil / putobject obj / putstring "..." + * putself / putnil / putobject obj / dupstring "..." * pop * => * # do nothing @@ -3539,7 +3771,8 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal if (IS_INSN(prev)) { enum ruby_vminsn_type previ = ((INSN *)prev)->insn_id; if (previ == BIN(putobject) || previ == BIN(putnil) || - previ == BIN(putself) || previ == BIN(putstring) || + previ == BIN(putself) || previ == BIN(dupstring) || + previ == BIN(dupchilledstring) || previ == BIN(dup) || previ == BIN(getlocal) || previ == BIN(getblockparam) || @@ -3681,10 +3914,10 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal } } - if (IS_INSN_ID(iobj, putstring) || + if (IS_INSN_ID(iobj, dupstring) || IS_INSN_ID(iobj, dupchilledstring) || (IS_INSN_ID(iobj, putobject) && RB_TYPE_P(OPERAND_AT(iobj, 0), T_STRING))) { /* - * putstring "" + * dupstring "" * concatstrings N * => * concatstrings N-1 @@ -3697,6 +3930,27 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal } ELEM_REMOVE(&iobj->link); } + if (IS_NEXT_INSN_ID(&iobj->link, toregexp)) { + INSN *next = (INSN *)iobj->link.next; + if (OPERAND_AT(next, 1) == INT2FIX(1)) { + VALUE src = OPERAND_AT(iobj, 0); + int opt = (int)FIX2LONG(OPERAND_AT(next, 0)); + VALUE path = rb_iseq_path(iseq); + int line = iobj->insn_info.line_no; + VALUE errinfo = rb_errinfo(); + VALUE re = rb_reg_compile(src, opt, RSTRING_PTR(path), line); + if (NIL_P(re)) { + VALUE message = rb_attr_get(rb_errinfo(), idMesg); + rb_set_errinfo(errinfo); + COMPILE_ERROR(iseq, line, "%" PRIsVALUE, message); + } + else { + RB_OBJ_SET_SHAREABLE(re); + } + RB_OBJ_WRITE(iseq, &OPERAND_AT(iobj, 0), re); + ELEM_REMOVE(iobj->link.next); + } + } } if (IS_INSN_ID(iobj, concatstrings)) { @@ -3734,8 +3988,6 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal if (do_tailcallopt && (IS_INSN_ID(iobj, send) || - IS_INSN_ID(iobj, opt_aref_with) || - IS_INSN_ID(iobj, opt_aset_with) || IS_INSN_ID(iobj, invokesuper))) { /* * send ... @@ -3869,8 +4121,8 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal if (IS_NEXT_INSN_ID(iobj->link.next, leave)) { iobj->insn_id = BIN(opt_invokebuiltin_delegate_leave); const struct rb_builtin_function *bf = (const struct rb_builtin_function *)iobj->operands[0]; - if (iobj == (INSN *)list && bf->argc == 0 && (iseq->body->builtin_attrs & BUILTIN_ATTR_LEAF)) { - iseq->body->builtin_attrs |= BUILTIN_ATTR_SINGLE_NOARG_LEAF; + if (iobj == (INSN *)list && bf->argc == 0 && (ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_LEAF)) { + ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_SINGLE_NOARG_LEAF; } } } @@ -3889,134 +4141,63 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal } } - if (IS_INSN_ID(iobj, splatarray) && OPERAND_AT(iobj, 0) == Qtrue) { + if (IS_INSN_ID(iobj, splatarray) && OPERAND_AT(iobj, 0) == false) { LINK_ELEMENT *niobj = &iobj->link; - - /* - * Eliminate array allocation for f(1, *a) - * - * splatarray true - * send ARGS_SPLAT and not KW_SPLAT|ARGS_BLOCKARG - * => - * splatarray false - * send - */ - if (optimize_args_splat_no_copy(iseq, iobj, niobj, - VM_CALL_ARGS_SPLAT, VM_CALL_KW_SPLAT|VM_CALL_ARGS_BLOCKARG, 0)) goto optimized_splat; - - if (IS_NEXT_INSN_ID(niobj, getlocal) || IS_NEXT_INSN_ID(niobj, getinstancevariable)) { + if (IS_NEXT_INSN_ID(niobj, duphash)) { niobj = niobj->next; + LINK_ELEMENT *siobj; + unsigned int set_flags = 0, unset_flags = 0; /* - * Eliminate array allocation for f(1, *a, &lvar) and f(1, *a, &@iv) + * Eliminate hash allocation for f(*a, kw: 1) * - * splatarray true - * getlocal / getinstancevariable - * send ARGS_SPLAT|ARGS_BLOCKARG and not KW_SPLAT - * => * splatarray false - * getlocal / getinstancevariable - * send - */ - if (optimize_args_splat_no_copy(iseq, iobj, niobj, - VM_CALL_ARGS_SPLAT|VM_CALL_ARGS_BLOCKARG, VM_CALL_KW_SPLAT, 0)) goto optimized_splat; - - /* - * Eliminate array allocation for f(*a, **lvar) and f(*a, **@iv) - * - * splatarray true - * getlocal / getinstancevariable - * send ARGS_SPLAT|KW_SPLAT and not ARGS_BLOCKARG + * duphash + * send ARGS_SPLAT|KW_SPLAT|KW_SPLAT_MUT and not ARGS_BLOCKARG * => * splatarray false - * getlocal / getinstancevariable - * send + * putobject + * send ARGS_SPLAT|KW_SPLAT */ - if (optimize_args_splat_no_copy(iseq, iobj, niobj, - VM_CALL_ARGS_SPLAT|VM_CALL_KW_SPLAT, VM_CALL_ARGS_BLOCKARG, 0)) goto optimized_splat; - - if (IS_NEXT_INSN_ID(niobj, getlocal) || IS_NEXT_INSN_ID(niobj, getinstancevariable) || - IS_NEXT_INSN_ID(niobj, getblockparamproxy)) { - niobj = niobj->next; - - /* - * Eliminate array allocation for f(*a, **lvar, &{arg,lvar,@iv}) - * - * splatarray true - * getlocal / getinstancevariable - * getlocal / getinstancevariable / getblockparamproxy - * send ARGS_SPLAT|KW_SPLAT|ARGS_BLOCKARG - * => - * splatarray false - * getlocal / getinstancevariable - * getlocal / getinstancevariable / getblockparamproxy - * send - */ - optimize_args_splat_no_copy(iseq, iobj, niobj, - VM_CALL_ARGS_SPLAT|VM_CALL_KW_SPLAT|VM_CALL_ARGS_BLOCKARG, 0, 0); - } - } else if (IS_NEXT_INSN_ID(niobj, getblockparamproxy)) { + if (IS_NEXT_INSN_ID(niobj, send)) { + siobj = niobj->next; + set_flags = VM_CALL_ARGS_SPLAT|VM_CALL_KW_SPLAT|VM_CALL_KW_SPLAT_MUT; + unset_flags = VM_CALL_ARGS_BLOCKARG; + } /* - * Eliminate array allocation for f(1, *a, &arg) + * Eliminate hash allocation for f(*a, kw: 1, &{arg,lvar,@iv}) * - * splatarray true - * getblockparamproxy - * send ARGS_SPLAT|ARGS_BLOCKARG and not KW_SPLAT - * => * splatarray false - * getblockparamproxy - * send - */ - optimize_args_splat_no_copy(iseq, iobj, niobj, - VM_CALL_ARGS_SPLAT|VM_CALL_ARGS_BLOCKARG, VM_CALL_KW_SPLAT, 0); - } else if (IS_NEXT_INSN_ID(niobj, duphash)) { - niobj = niobj->next; - - /* - * Eliminate array and hash allocation for f(*a, kw: 1) - * - * splatarray true * duphash - * send ARGS_SPLAT|KW_SPLAT|KW_SPLAT_MUT and not ARGS_BLOCKARG + * getlocal / getinstancevariable / getblockparamproxy + * send ARGS_SPLAT|KW_SPLAT|KW_SPLAT_MUT|ARGS_BLOCKARG * => * splatarray false * putobject - * send ARGS_SPLAT|KW_SPLAT + * getlocal / getinstancevariable / getblockparamproxy + * send ARGS_SPLAT|KW_SPLAT|ARGS_BLOCKARG */ - if (optimize_args_splat_no_copy(iseq, iobj, niobj, - VM_CALL_ARGS_SPLAT|VM_CALL_KW_SPLAT|VM_CALL_KW_SPLAT_MUT, VM_CALL_ARGS_BLOCKARG, VM_CALL_KW_SPLAT_MUT)) { - - ((INSN*)niobj)->insn_id = BIN(putobject); - OPERAND_AT(niobj, 0) = rb_hash_freeze(rb_hash_resurrect(OPERAND_AT(niobj, 0))); - - goto optimized_splat; + else if ((IS_NEXT_INSN_ID(niobj, getlocal) || IS_NEXT_INSN_ID(niobj, getinstancevariable) || + IS_NEXT_INSN_ID(niobj, getblockparamproxy)) && (IS_NEXT_INSN_ID(niobj->next, send))) { + siobj = niobj->next->next; + set_flags = VM_CALL_ARGS_SPLAT|VM_CALL_KW_SPLAT|VM_CALL_KW_SPLAT_MUT|VM_CALL_ARGS_BLOCKARG; } - if (IS_NEXT_INSN_ID(niobj, getlocal) || IS_NEXT_INSN_ID(niobj, getinstancevariable) || - IS_NEXT_INSN_ID(niobj, getblockparamproxy)) { - /* - * Eliminate array and hash allocation for f(*a, kw: 1, &{arg,lvar,@iv}) - * - * splatarray true - * duphash - * getlocal / getinstancevariable / getblockparamproxy - * send ARGS_SPLAT|KW_SPLAT|KW_SPLAT_MUT|ARGS_BLOCKARG - * => - * splatarray false - * putobject - * getlocal / getinstancevariable / getblockparamproxy - * send ARGS_SPLAT|KW_SPLAT|ARGS_BLOCKARG - */ - if (optimize_args_splat_no_copy(iseq, iobj, niobj->next, - VM_CALL_ARGS_SPLAT|VM_CALL_KW_SPLAT|VM_CALL_KW_SPLAT_MUT|VM_CALL_ARGS_BLOCKARG, 0, VM_CALL_KW_SPLAT_MUT)) { - + if (set_flags) { + const struct rb_callinfo *ci = (const struct rb_callinfo *)OPERAND_AT(siobj, 0); + unsigned int flags = vm_ci_flag(ci); + if ((flags & set_flags) == set_flags && !(flags & unset_flags)) { ((INSN*)niobj)->insn_id = BIN(putobject); - OPERAND_AT(niobj, 0) = rb_hash_freeze(rb_hash_resurrect(OPERAND_AT(niobj, 0))); + RB_OBJ_WRITE(iseq, &OPERAND_AT(niobj, 0), RB_OBJ_SET_SHAREABLE(rb_hash_freeze(rb_hash_resurrect(OPERAND_AT(niobj, 0))))); + + const struct rb_callinfo *nci = vm_ci_new(vm_ci_mid(ci), + flags & ~VM_CALL_KW_SPLAT_MUT, vm_ci_argc(ci), vm_ci_kwarg(ci)); + RB_OBJ_WRITTEN(iseq, ci, nci); + OPERAND_AT(siobj, 0) = (VALUE)nci; } } } } - optimized_splat: return COMPILE_OK; } @@ -4024,17 +4205,16 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal static int insn_set_specialized_instruction(rb_iseq_t *iseq, INSN *iobj, int insn_id) { - iobj->insn_id = insn_id; - iobj->operand_size = insn_len(insn_id) - 1; - iobj->insn_info.events |= RUBY_EVENT_C_CALL | RUBY_EVENT_C_RETURN; - if (insn_id == BIN(opt_neq)) { VALUE original_ci = iobj->operands[0]; - iobj->operand_size = 2; - iobj->operands = compile_data_calloc2(iseq, iobj->operand_size, sizeof(VALUE)); - iobj->operands[0] = (VALUE)new_callinfo(iseq, idEq, 1, 0, NULL, FALSE); - iobj->operands[1] = original_ci; + VALUE new_ci = (VALUE)new_callinfo(iseq, idEq, 1, 0, NULL, FALSE); + insn_replace_with_operands(iseq, iobj, insn_id, 2, new_ci, original_ci); + } + else { + iobj->insn_id = insn_id; + iobj->operand_size = insn_len(insn_id) - 1; } + iobj->insn_info.events |= RUBY_EVENT_C_CALL | RUBY_EVENT_C_RETURN; return COMPILE_OK; } @@ -4045,37 +4225,145 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) if (IS_INSN_ID(iobj, newarray) && iobj->link.next && IS_INSN(iobj->link.next)) { /* - * [a, b, ...].max/min -> a, b, c, opt_newarray_max/min + * [a, b, ...].max/min -> a, b, c, opt_newarray_send max/min */ INSN *niobj = (INSN *)iobj->link.next; if (IS_INSN_ID(niobj, send)) { const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(niobj, 0); - if ((vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) && vm_ci_argc(ci) == 0) { + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 0) { + VALUE method = INT2FIX(0); switch (vm_ci_mid(ci)) { case idMax: + method = INT2FIX(VM_OPT_NEWARRAY_SEND_MAX); + break; case idMin: + method = INT2FIX(VM_OPT_NEWARRAY_SEND_MIN); + break; case idHash: - { - VALUE num = iobj->operands[0]; - iobj->insn_id = BIN(opt_newarray_send); - iobj->operands = compile_data_calloc2(iseq, insn_len(iobj->insn_id) - 1, sizeof(VALUE)); - iobj->operands[0] = num; - iobj->operands[1] = rb_id2sym(vm_ci_mid(ci)); - iobj->operand_size = insn_len(iobj->insn_id) - 1; - ELEM_REMOVE(&niobj->link); - return COMPILE_OK; - } + method = INT2FIX(VM_OPT_NEWARRAY_SEND_HASH); + break; } + + if (method != INT2FIX(0)) { + VALUE num = iobj->operands[0]; + insn_replace_with_operands(iseq, iobj, BIN(opt_newarray_send), 2, num, method); + ELEM_REMOVE(&niobj->link); + return COMPILE_OK; + } + } + } + else if ((IS_INSN_ID(niobj, dupstring) || IS_INSN_ID(niobj, dupchilledstring) || + (IS_INSN_ID(niobj, putobject) && RB_TYPE_P(OPERAND_AT(niobj, 0), T_STRING))) && + IS_NEXT_INSN_ID(&niobj->link, send)) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT((INSN *)niobj->link.next, 0); + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 1 && vm_ci_mid(ci) == idPack) { + VALUE num = iobj->operands[0]; + insn_replace_with_operands(iseq, iobj, BIN(opt_newarray_send), 2, FIXNUM_INC(num, 1), INT2FIX(VM_OPT_NEWARRAY_SEND_PACK)); + ELEM_REMOVE(&iobj->link); + ELEM_REMOVE(niobj->link.next); + ELEM_INSERT_NEXT(&niobj->link, &iobj->link); + return COMPILE_OK; + } + } + // newarray n, dupchilledstring "E", getlocal b, send :pack with {buffer: b} + // -> dupchilledstring "E", getlocal b, opt_newarray_send n+2, :pack, :buffer + else if ((IS_INSN_ID(niobj, dupstring) || IS_INSN_ID(niobj, dupchilledstring) || + (IS_INSN_ID(niobj, putobject) && RB_TYPE_P(OPERAND_AT(niobj, 0), T_STRING))) && + IS_NEXT_INSN_ID(&niobj->link, getlocal) && + (niobj->link.next && IS_NEXT_INSN_ID(niobj->link.next, send))) { + const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT((INSN *)(niobj->link.next)->next, 0); + const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci); + if (vm_ci_mid(ci) == idPack && vm_ci_argc(ci) == 2 && + (kwarg && kwarg->keyword_len == 1 && kwarg->keywords[0] == rb_id2sym(idBuffer))) { + VALUE num = iobj->operands[0]; + insn_replace_with_operands(iseq, iobj, BIN(opt_newarray_send), 2, FIXNUM_INC(num, 2), INT2FIX(VM_OPT_NEWARRAY_SEND_PACK_BUFFER)); + // Remove the "send" insn. + ELEM_REMOVE((niobj->link.next)->next); + // Remove the modified insn from its original "newarray" position... + ELEM_REMOVE(&iobj->link); + // and insert it after the buffer insn. + ELEM_INSERT_NEXT(niobj->link.next, &iobj->link); + return COMPILE_OK; + } + } + + // Break the "else if" chain since some prior checks abort after sub-ifs. + // We already found "newarray". To match `[...].include?(arg)` we look for + // the instruction(s) representing the argument followed by a "send". + if ((IS_INSN_ID(niobj, dupstring) || IS_INSN_ID(niobj, dupchilledstring) || + IS_INSN_ID(niobj, putobject) || + IS_INSN_ID(niobj, putself) || + IS_INSN_ID(niobj, getlocal) || + IS_INSN_ID(niobj, getinstancevariable)) && + IS_NEXT_INSN_ID(&niobj->link, send)) { + + LINK_ELEMENT *sendobj = &(niobj->link); // Below we call ->next; + const struct rb_callinfo *ci; + // Allow any number (0 or more) of simple method calls on the argument + // (as in `[...].include?(arg.method1.method2)`. + do { + sendobj = sendobj->next; + ci = (struct rb_callinfo *)OPERAND_AT(sendobj, 0); + } while (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && IS_NEXT_INSN_ID(sendobj, send)); + + // If this send is for .include? with one arg we can do our opt. + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 1 && vm_ci_mid(ci) == idIncludeP) { + VALUE num = iobj->operands[0]; + INSN *sendins = (INSN *)sendobj; + insn_replace_with_operands(iseq, sendins, BIN(opt_newarray_send), 2, FIXNUM_INC(num, 1), INT2FIX(VM_OPT_NEWARRAY_SEND_INCLUDE_P)); + // Remove the original "newarray" insn. + ELEM_REMOVE(&iobj->link); + return COMPILE_OK; + } + } + } + + /* + * duparray [...] + * some insn for the arg... + * send <calldata!mid:include?, argc:1, ARGS_SIMPLE>, nil + * => + * arg insn... + * opt_duparray_send [...], :include?, 1 + */ + if (IS_INSN_ID(iobj, duparray) && iobj->link.next && IS_INSN(iobj->link.next)) { + INSN *niobj = (INSN *)iobj->link.next; + if ((IS_INSN_ID(niobj, getlocal) || + IS_INSN_ID(niobj, getinstancevariable) || + IS_INSN_ID(niobj, putself)) && + IS_NEXT_INSN_ID(&niobj->link, send)) { + + LINK_ELEMENT *sendobj = &(niobj->link); // Below we call ->next; + const struct rb_callinfo *ci; + // Allow any number (0 or more) of simple method calls on the argument + // (as in `[...].include?(arg.method1.method2)`. + do { + sendobj = sendobj->next; + ci = (struct rb_callinfo *)OPERAND_AT(sendobj, 0); + } while (vm_ci_simple(ci) && vm_ci_argc(ci) == 0 && IS_NEXT_INSN_ID(sendobj, send)); + + if (vm_ci_simple(ci) && vm_ci_argc(ci) == 1 && vm_ci_mid(ci) == idIncludeP) { + // Move the array arg from duparray to opt_duparray_send. + VALUE ary = iobj->operands[0]; + rb_obj_reveal(ary, rb_cArray); + + INSN *sendins = (INSN *)sendobj; + insn_replace_with_operands(iseq, sendins, BIN(opt_duparray_send), 3, ary, rb_id2sym(idIncludeP), INT2FIX(1)); + + // Remove the duparray insn. + ELEM_REMOVE(&iobj->link); + return COMPILE_OK; } } } + if (IS_INSN_ID(iobj, send)) { const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(iobj, 0); const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(iobj, 1); #define SP_INSN(opt) insn_set_specialized_instruction(iseq, iobj, BIN(opt_##opt)) - if (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE) { + if (vm_ci_simple(ci)) { switch (vm_ci_argc(ci)) { case 0: switch (vm_ci_mid(ci)) { @@ -4115,7 +4403,7 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) } } - if ((vm_ci_flag(ci) & VM_CALL_ARGS_BLOCKARG) == 0 && blockiseq == NULL) { + if ((vm_ci_flag(ci) & (VM_CALL_ARGS_BLOCKARG | VM_CALL_FORWARDING)) == 0 && blockiseq == NULL) { iobj->insn_id = BIN(opt_send_without_block); iobj->operand_size = insn_len(iobj->insn_id) - 1; } @@ -4151,15 +4439,25 @@ iseq_optimize(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) ISEQ_COMPILE_DATA(iseq)->option->tailcall_optimization; const int do_si = ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction; const int do_ou = ISEQ_COMPILE_DATA(iseq)->option->operands_unification; + const int do_without_ints = ISEQ_BODY(iseq)->builtin_attrs & BUILTIN_ATTR_WITHOUT_INTERRUPTS; int rescue_level = 0; int tailcallopt = do_tailcallopt; list = FIRST_ELEMENT(anchor); int do_block_optimization = 0; + LABEL * block_loop_label = NULL; - if (ISEQ_BODY(iseq)->type == ISEQ_TYPE_BLOCK && !ISEQ_COMPILE_DATA(iseq)->catch_except_p) { + // If we're optimizing a block + if (ISEQ_BODY(iseq)->type == ISEQ_TYPE_BLOCK) { do_block_optimization = 1; + + // If the block starts with a nop and a label, + // record the label so we can detect if it's a jump target + LINK_ELEMENT * le = FIRST_ELEMENT(anchor)->next; + if (IS_INSN(le) && IS_INSN_ID((INSN *)le, nop) && IS_LABEL(le->next)) { + block_loop_label = (LABEL *)le->next; + } } while (list) { @@ -4174,11 +4472,45 @@ iseq_optimize(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) insn_operands_unification((INSN *)list); } + if (do_without_ints) { + INSN *item = (INSN *)list; + if (IS_INSN_ID(item, jump)) { + item->insn_id = BIN(jump_without_ints); + } + else if (IS_INSN_ID(item, branchif)) { + item->insn_id = BIN(branchif_without_ints); + } + else if (IS_INSN_ID(item, branchunless)) { + item->insn_id = BIN(branchunless_without_ints); + } + else if (IS_INSN_ID(item, branchnil)) { + item->insn_id = BIN(branchnil_without_ints); + } + } + if (do_block_optimization) { INSN * item = (INSN *)list; - if (IS_INSN_ID(item, jump)) { + // Give up if there is a throw + if (IS_INSN_ID(item, throw)) { do_block_optimization = 0; } + else { + // If the instruction has a jump target, check if the + // jump target is the block loop label + const char *types = insn_op_types(item->insn_id); + for (int j = 0; types[j]; j++) { + if (types[j] == TS_OFFSET) { + // If the jump target is equal to the block loop + // label, then we can't do the optimization because + // the leading `nop` instruction fires the block + // entry tracepoint + LABEL * target = (LABEL *)OPERAND_AT(item, j); + if (target == block_loop_label) { + do_block_optimization = 0; + } + } + } + } } } if (IS_LABEL(list)) { @@ -4223,7 +4555,7 @@ new_unified_insn(rb_iseq_t *iseq, } if (argc > 0) { - ptr = operands = compile_data_alloc2(iseq, sizeof(VALUE), argc); + ptr = operands = compile_data_alloc2_type(iseq, VALUE, argc); } /* copy operands */ @@ -4318,47 +4650,91 @@ all_string_result_p(const NODE *node) } } +struct dstr_ctxt { + rb_iseq_t *const iseq; + LINK_ANCHOR *const ret; + VALUE lit; + const NODE *lit_node; + int cnt; + int dregx; +}; + static int -compile_dstr_fragments(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int *cntp) +append_dstr_fragment(struct dstr_ctxt *args, const NODE *const node, rb_parser_string_t *str) { - const struct RNode_LIST *list = RNODE_DSTR(node)->nd_next; - VALUE lit = rb_node_dstr_string_val(node); - LINK_ELEMENT *first_lit = 0; - int cnt = 0; - - debugp_param("nd_lit", lit); - if (!NIL_P(lit)) { - cnt++; - if (!RB_TYPE_P(lit, T_STRING)) { - COMPILE_ERROR(ERROR_ARGS "dstr: must be string: %s", - rb_builtin_type_name(TYPE(lit))); + VALUE s = rb_str_new_mutable_parser_string(str); + if (args->dregx) { + VALUE error = rb_reg_check_preprocess(s); + if (!NIL_P(error)) { + COMPILE_ERROR(args->iseq, nd_line(node), "%" PRIsVALUE, error); return COMPILE_NG; } + } + if (NIL_P(args->lit)) { + args->lit = s; + args->lit_node = node; + } + else { + rb_str_buf_append(args->lit, s); + } + return COMPILE_OK; +} + +static void +flush_dstr_fragment(struct dstr_ctxt *args) +{ + if (!NIL_P(args->lit)) { + rb_iseq_t *iseq = args->iseq; + VALUE lit = args->lit; + args->lit = Qnil; lit = rb_fstring(lit); - ADD_INSN1(ret, node, putobject, lit); - RB_OBJ_WRITTEN(iseq, Qundef, lit); - if (RSTRING_LEN(lit) == 0) first_lit = LAST_ELEMENT(ret); + ADD_INSN1(args->ret, args->lit_node, putobject, lit); + RB_OBJ_WRITTEN(args->iseq, Qundef, lit); + args->cnt++; + } +} + +static int +compile_dstr_fragments_0(struct dstr_ctxt *args, const NODE *const node) +{ + const struct RNode_LIST *list = RNODE_DSTR(node)->nd_next; + rb_parser_string_t *str = RNODE_DSTR(node)->string; + + if (str) { + CHECK(append_dstr_fragment(args, node, str)); } while (list) { const NODE *const head = list->nd_head; if (nd_type_p(head, NODE_STR)) { - lit = rb_fstring(rb_node_str_string_val(head)); - ADD_INSN1(ret, head, putobject, lit); - RB_OBJ_WRITTEN(iseq, Qundef, lit); - lit = Qnil; + CHECK(append_dstr_fragment(args, node, RNODE_STR(head)->string)); + } + else if (nd_type_p(head, NODE_DSTR)) { + CHECK(compile_dstr_fragments_0(args, head)); } else { - CHECK(COMPILE(ret, "each string", head)); + flush_dstr_fragment(args); + rb_iseq_t *iseq = args->iseq; + CHECK(COMPILE(args->ret, "each string", head)); + args->cnt++; } - cnt++; list = (struct RNode_LIST *)list->nd_next; } - if (NIL_P(lit) && first_lit) { - ELEM_REMOVE(first_lit); - --cnt; - } - *cntp = cnt; + return COMPILE_OK; +} + +static int +compile_dstr_fragments(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int *cntp, int dregx) +{ + struct dstr_ctxt args = { + .iseq = iseq, .ret = ret, + .lit = Qnil, .lit_node = NULL, + .cnt = 0, .dregx = dregx, + }; + CHECK(compile_dstr_fragments_0(&args, node)); + flush_dstr_fragment(&args); + + *cntp = args.cnt; return COMPILE_OK; } @@ -4382,12 +4758,13 @@ compile_dstr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node) { int cnt; if (!RNODE_DSTR(node)->nd_next) { - VALUE lit = rb_fstring(rb_node_dstr_string_val(node)); - ADD_INSN1(ret, node, putstring, lit); + VALUE lit = rb_node_dstr_string_val(node); + ADD_INSN1(ret, node, dupstring, lit); + RB_OBJ_SET_SHAREABLE(lit); RB_OBJ_WRITTEN(iseq, Qundef, lit); } else { - CHECK(compile_dstr_fragments(iseq, ret, node, &cnt)); + CHECK(compile_dstr_fragments(iseq, ret, node, &cnt, FALSE)); ADD_INSN1(ret, node, concatstrings, INT2FIX(cnt)); } return COMPILE_OK; @@ -4397,19 +4774,21 @@ static int compile_dregx(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { int cnt; + int cflag = (int)RNODE_DREGX(node)->as.nd_cflag; if (!RNODE_DREGX(node)->nd_next) { if (!popped) { VALUE src = rb_node_dregx_string_val(node); - VALUE match = rb_reg_compile(src, (int)RNODE_DREGX(node)->nd_cflag, NULL, 0); + VALUE match = rb_reg_compile(src, cflag, NULL, 0); + RB_OBJ_SET_SHAREABLE(match); ADD_INSN1(ret, node, putobject, match); RB_OBJ_WRITTEN(iseq, Qundef, match); } return COMPILE_OK; } - CHECK(compile_dstr_fragments(iseq, ret, node, &cnt)); - ADD_INSN2(ret, node, toregexp, INT2FIX(RNODE_DREGX(node)->nd_cflag), INT2FIX(cnt)); + CHECK(compile_dstr_fragments(iseq, ret, node, &cnt, TRUE)); + ADD_INSN2(ret, node, toregexp, INT2FIX(cflag), INT2FIX(cnt)); if (popped) { ADD_INSN(ret, node, pop); @@ -4494,6 +4873,7 @@ compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *ret, const NODE *cond, CHECK(ok = compile_logical(iseq, ret, RNODE_AND(cond)->nd_1st, NULL, else_label)); cond = RNODE_AND(cond)->nd_2nd; if (ok == COMPILE_SINGLE) { + ADD_INSNL(ret, cond, jump, else_label); INIT_ANCHOR(ignore); ret = ignore; then_label = NEW_LABEL(nd_line(cond)); @@ -4503,12 +4883,12 @@ compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *ret, const NODE *cond, CHECK(ok = compile_logical(iseq, ret, RNODE_OR(cond)->nd_1st, then_label, NULL)); cond = RNODE_OR(cond)->nd_2nd; if (ok == COMPILE_SINGLE) { + ADD_INSNL(ret, cond, jump, then_label); INIT_ANCHOR(ignore); ret = ignore; else_label = NEW_LABEL(nd_line(cond)); } goto again; - case NODE_LIT: /* NODE_LIT is always true */ case NODE_SYM: case NODE_LINE: case NODE_FILE: @@ -4544,7 +4924,7 @@ compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *ret, const NODE *cond, CHECK(compile_flip_flop(iseq, ret, cond, FALSE, then_label, else_label)); return COMPILE_OK; case NODE_DEFINED: - CHECK(compile_defined_expr(iseq, ret, cond, Qfalse)); + CHECK(compile_defined_expr(iseq, ret, cond, Qfalse, ret == ignore)); break; default: { @@ -4589,8 +4969,6 @@ static VALUE get_symbol_value(rb_iseq_t *iseq, const NODE *node) { switch (nd_type(node)) { - case NODE_LIT: - return RNODE_LIT(node)->nd_lit; case NODE_SYM: return rb_node_sym_string_val(node); default: @@ -4639,10 +5017,7 @@ compile_keyword_arg(rb_iseq_t *iseq, LINK_ANCHOR *const ret, seen_nodes++; RUBY_ASSERT(nd_type_p(node, NODE_LIST)); - if (key_node && nd_type_p(key_node, NODE_LIT) && SYMBOL_P(RNODE_LIT(key_node)->nd_lit)) { - /* can be keywords */ - } - else if (key_node && nd_type_p(key_node, NODE_SYM)) { + if (key_node && nd_type_p(key_node, NODE_SYM)) { /* can be keywords */ } else { @@ -4727,7 +5102,6 @@ static inline bool static_literal_node_p(const NODE *node, const rb_iseq_t *iseq, bool hash_key) { switch (nd_type(node)) { - case NODE_LIT: case NODE_SYM: case NODE_REGX: case NODE_LINE: @@ -4753,13 +5127,21 @@ static_literal_value(const NODE *node, rb_iseq_t *iseq) { switch (nd_type(node)) { case NODE_INTEGER: - return rb_node_integer_literal_val(node); + { + VALUE lit = rb_node_integer_literal_val(node); + if (!SPECIAL_CONST_P(lit)) RB_OBJ_SET_SHAREABLE(lit); + return lit; + } case NODE_FLOAT: - return rb_node_float_literal_val(node); + { + VALUE lit = rb_node_float_literal_val(node); + if (!SPECIAL_CONST_P(lit)) RB_OBJ_SET_SHAREABLE(lit); + return lit; + } case NODE_RATIONAL: - return rb_node_rational_literal_val(node); + return rb_ractor_make_shareable(rb_node_rational_literal_val(node)); case NODE_IMAGINARY: - return rb_node_imaginary_literal_val(node); + return rb_ractor_make_shareable(rb_node_imaginary_literal_val(node)); case NODE_NIL: return Qnil; case NODE_TRUE: @@ -4769,7 +5151,7 @@ static_literal_value(const NODE *node, rb_iseq_t *iseq) case NODE_SYM: return rb_node_sym_string_val(node); case NODE_REGX: - return rb_node_regx_string_val(node); + return RB_OBJ_SET_SHAREABLE(rb_node_regx_string_val(node)); case NODE_LINE: return rb_node_line_lineno_val(node); case NODE_ENCODING: @@ -4777,17 +5159,14 @@ static_literal_value(const NODE *node, rb_iseq_t *iseq) case NODE_FILE: case NODE_STR: if (ISEQ_COMPILE_DATA(iseq)->option->debug_frozen_string_literal || RTEST(ruby_debug)) { - VALUE lit; - VALUE debug_info = rb_ary_new_from_args(2, rb_iseq_path(iseq), INT2FIX((int)nd_line(node))); - lit = rb_str_dup(get_string_value(node)); - rb_ivar_set(lit, id_debug_created_info, rb_obj_freeze(debug_info)); - return rb_str_freeze(lit); + VALUE lit = get_string_value(node); + VALUE str = rb_str_with_debug_created_info(lit, rb_iseq_path(iseq), (int)nd_line(node)); + RB_OBJ_SET_SHAREABLE(str); + return str; } else { - return rb_fstring(get_string_value(node)); + return get_string_value(node); } - case NODE_LIT: - return RNODE_LIT(node)->nd_lit; default: rb_bug("unexpected node: %s", ruby_node_name(nd_type(node))); } @@ -4881,7 +5260,7 @@ compile_array(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, int pop /* Create a hidden array */ for (; count; count--, node = RNODE_LIST(node)->nd_next) rb_ary_push(ary, static_literal_value(RNODE_LIST(node)->nd_head, iseq)); - OBJ_FREEZE(ary); + RB_OBJ_SET_FROZEN_SHAREABLE(ary); /* Emit optimized code */ FLUSH_CHUNK; @@ -4893,6 +5272,7 @@ compile_array(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, int pop ADD_INSN1(ret, line_node, putobject, ary); ADD_INSN(ret, line_node, concattoarray); } + RB_OBJ_SET_SHAREABLE(ary); RB_OBJ_WRITTEN(iseq, Qundef, ary); } } @@ -5019,13 +5399,15 @@ compile_hash(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, int meth for (; count; count--, node = RNODE_LIST(RNODE_LIST(node)->nd_next)->nd_next) { VALUE elem[2]; elem[0] = static_literal_value(RNODE_LIST(node)->nd_head, iseq); + if (!RB_SPECIAL_CONST_P(elem[0])) RB_OBJ_SET_FROZEN_SHAREABLE(elem[0]); elem[1] = static_literal_value(RNODE_LIST(RNODE_LIST(node)->nd_next)->nd_head, iseq); + if (!RB_SPECIAL_CONST_P(elem[1])) RB_OBJ_SET_FROZEN_SHAREABLE(elem[1]); rb_ary_cat(ary, elem, 2); } - VALUE hash = rb_hash_new_with_size(RARRAY_LEN(ary) / 2); + VALUE hash = rb_hash_alloc_fixed_size(Qfalse, RARRAY_LEN(ary) / 2); rb_hash_bulk_insert(RARRAY_LEN(ary), RARRAY_CONST_PTR(ary), hash); - hash = rb_obj_hide(hash); - OBJ_FREEZE(hash); + RB_GC_GUARD(ary); + hash = RB_OBJ_SET_FROZEN_SHAREABLE(hash); /* Emit optimized code */ FLUSH_CHUNK(); @@ -5066,7 +5448,7 @@ compile_hash(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, int meth FLUSH_CHUNK(); const NODE *kw = RNODE_LIST(RNODE_LIST(node)->nd_next)->nd_head; - int empty_kw = nd_type_p(kw, NODE_LIT) && RB_TYPE_P(RNODE_LIT(kw)->nd_lit, T_HASH); /* foo( ..., **{}, ...) */ + int empty_kw = nd_type_p(kw, NODE_HASH) && (!RNODE_HASH(kw)->nd_head); /* foo( ..., **{}, ...) */ int first_kw = first_chunk && stack_len == 0; /* foo(1,2,3, **kw, ...) */ int last_kw = !RNODE_LIST(RNODE_LIST(node)->nd_next)->nd_next; /* foo( ..., **kw) */ int only_kw = last_kw && first_kw; /* foo(1,2,3, **kw) */ @@ -5131,13 +5513,6 @@ VALUE rb_node_case_when_optimizable_literal(const NODE *const node) { switch (nd_type(node)) { - case NODE_LIT: { - VALUE v = RNODE_LIT(node)->nd_lit; - if (SYMBOL_P(v)) { - return v; - } - break; - } case NODE_INTEGER: return rb_node_integer_literal_val(node); case NODE_FLOAT: { @@ -5163,9 +5538,9 @@ rb_node_case_when_optimizable_literal(const NODE *const node) case NODE_LINE: return rb_node_line_lineno_val(node); case NODE_STR: - return rb_fstring(rb_node_str_string_val(node)); + return rb_node_str_string_val(node); case NODE_FILE: - return rb_fstring(rb_node_file_path_val(node)); + return rb_node_file_path_val(node); } return Qundef; } @@ -5181,13 +5556,13 @@ when_vals(rb_iseq_t *iseq, LINK_ANCHOR *const cond_seq, const NODE *vals, if (UNDEF_P(lit)) { only_special_literals = 0; } - else if (NIL_P(rb_hash_lookup(literals, lit))) { - rb_hash_aset(literals, lit, (VALUE)(l1) | 1); + else { + cdhash_aset_if_missing(literals, lit, (VALUE)(l1)); } if (nd_type_p(val, NODE_STR) || nd_type_p(val, NODE_FILE)) { debugp_param("nd_lit", get_string_value(val)); - lit = rb_fstring(get_string_value(val)); + lit = get_string_value(val); ADD_INSN1(cond_seq, val, putobject, lit); RB_OBJ_WRITTEN(iseq, Qundef, lit); } @@ -5388,12 +5763,17 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const CHECK(COMPILE_POPPED(pre, "masgn lhs (NODE_ATTRASGN)", node)); + bool safenav_call = false; LINK_ELEMENT *insn_element = LAST_ELEMENT(pre); iobj = (INSN *)get_prev_insn((INSN *)insn_element); /* send insn */ ASSUME(iobj); - ELEM_REMOVE(LAST_ELEMENT(pre)); - ELEM_REMOVE((LINK_ELEMENT *)iobj); - pre->last = iobj->link.prev; + ELEM_REMOVE(insn_element); + if (!IS_INSN_ID(iobj, send)) { + safenav_call = true; + iobj = (INSN *)get_prev_insn(iobj); + ELEM_INSERT_NEXT(&iobj->link, insn_element); + } + (pre->last = iobj->link.prev)->next = 0; const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(iobj, 0); int argc = vm_ci_argc(ci) + 1; @@ -5412,7 +5792,9 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const return COMPILE_NG; } - ADD_ELEM(lhs, (LINK_ELEMENT *)iobj); + iobj->link.prev = lhs->last; + lhs->last->next = &iobj->link; + for (lhs->last = &iobj->link; lhs->last->next; lhs->last = lhs->last->next); if (vm_ci_flag(ci) & VM_CALL_ARGS_SPLAT) { int argc = vm_ci_argc(ci); bool dupsplat = false; @@ -5445,9 +5827,11 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const pre, LINK_ANCHOR *const } INSERT_BEFORE_INSN1(iobj, line_no, node_id, pushtoarray, INT2FIX(1)); } - ADD_INSN(lhs, line_node, pop); - if (argc != 1) { + if (!safenav_call) { ADD_INSN(lhs, line_node, pop); + if (argc != 1) { + ADD_INSN(lhs, line_node, pop); + } } for (int i=0; i < argc; i++) { ADD_INSN(post, line_node, pop); @@ -5690,10 +6074,12 @@ collect_const_segments(rb_iseq_t *iseq, const NODE *node) switch (nd_type(node)) { case NODE_CONST: rb_ary_unshift(arr, ID2SYM(RNODE_CONST(node)->nd_vid)); + RB_OBJ_SET_SHAREABLE(arr); return arr; case NODE_COLON3: rb_ary_unshift(arr, ID2SYM(RNODE_COLON3(node)->nd_mid)); rb_ary_unshift(arr, ID2SYM(idNULL)); + RB_OBJ_SET_SHAREABLE(arr); return arr; case NODE_COLON2: rb_ary_unshift(arr, ID2SYM(RNODE_COLON2(node)->nd_mid)); @@ -5736,6 +6122,23 @@ compile_const_prefix(rb_iseq_t *iseq, const NODE *const node, } static int +cpath_const_p(const NODE *node) +{ + switch (nd_type(node)) { + case NODE_CONST: + case NODE_COLON3: + return TRUE; + case NODE_COLON2: + if (RNODE_COLON2(node)->nd_head) { + return cpath_const_p(RNODE_COLON2(node)->nd_head); + } + return TRUE; + default: + return FALSE; + } +} + +static int compile_cpath(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const NODE *cpath) { if (nd_type_p(cpath, NODE_COLON3)) { @@ -5744,9 +6147,13 @@ compile_cpath(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const NODE *cpath) return VM_DEFINECLASS_FLAG_SCOPED; } else if (nd_type_p(cpath, NODE_COLON2) && RNODE_COLON2(cpath)->nd_head) { - /* Bar::Foo */ + /* Bar::Foo or expr::Foo */ NO_CHECK(COMPILE(ret, "nd_else->nd_head", RNODE_COLON2(cpath)->nd_head)); - return VM_DEFINECLASS_FLAG_SCOPED; + int flags = VM_DEFINECLASS_FLAG_SCOPED; + if (!cpath_const_p(RNODE_COLON2(cpath)->nd_head)) { + flags |= VM_DEFINECLASS_FLAG_DYNAMIC_CREF; + } + return flags; } else { /* class at cbase Foo */ @@ -5768,7 +6175,7 @@ private_recv_p(const NODE *node) static void defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, - const NODE *const node, LABEL **lfinish, VALUE needstr); + const NODE *const node, LABEL **lfinish, VALUE needstr, bool ignore); static int compile_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, const enum node_type type, const NODE *const line_node, int popped, bool assume_receiver); @@ -5799,21 +6206,25 @@ defined_expr0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, expr_type = DEFINED_FALSE; break; + case NODE_HASH: case NODE_LIST:{ - const NODE *vals = node; + const NODE *vals = (nd_type(node) == NODE_HASH) ? RNODE_HASH(node)->nd_head : node; - do { - defined_expr0(iseq, ret, RNODE_LIST(vals)->nd_head, lfinish, Qfalse, false); + if (vals) { + do { + if (RNODE_LIST(vals)->nd_head) { + defined_expr0(iseq, ret, RNODE_LIST(vals)->nd_head, lfinish, Qfalse, false); - if (!lfinish[1]) { - lfinish[1] = NEW_LABEL(line); - } - ADD_INSNL(ret, line_node, branchunless, lfinish[1]); - } while ((vals = RNODE_LIST(vals)->nd_next) != NULL); + if (!lfinish[1]) { + lfinish[1] = NEW_LABEL(line); + } + ADD_INSNL(ret, line_node, branchunless, lfinish[1]); + } + } while ((vals = RNODE_LIST(vals)->nd_next) != NULL); + } } /* fall through */ case NODE_STR: - case NODE_LIT: case NODE_SYM: case NODE_REGX: case NODE_LINE: @@ -5830,6 +6241,15 @@ defined_expr0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, expr_type = DEFINED_EXPR; break; + case NODE_SPLAT: + defined_expr0(iseq, ret, RNODE_LIST(node)->nd_head, lfinish, Qfalse, false); + if (!lfinish[1]) { + lfinish[1] = NEW_LABEL(line); + } + ADD_INSNL(ret, line_node, branchunless, lfinish[1]); + expr_type = DEFINED_EXPR; + break; + /* variables */ case NODE_LVAR: case NODE_DVAR: @@ -5941,6 +6361,7 @@ defined_expr0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, ADD_INSN(ret, line_node, putnil); ADD_INSN3(ret, line_node, defined, INT2FIX(DEFINED_YIELD), 0, PUSH_VAL(DEFINED_YIELD)); + iseq_set_use_block(ISEQ_BODY(iseq)->local_iseq); return; case NODE_BACK_REF: @@ -5995,7 +6416,7 @@ build_defined_rescue_iseq(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const void *u static void defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, - const NODE *const node, LABEL **lfinish, VALUE needstr) + const NODE *const node, LABEL **lfinish, VALUE needstr, bool ignore) { LINK_ELEMENT *lcur = ret->last; defined_expr0(iseq, ret, node, lfinish, needstr, false); @@ -6006,20 +6427,22 @@ defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const rb_iseq_t *rescue; struct rb_iseq_new_with_callback_callback_func *ifunc = rb_iseq_new_with_callback_new_callback(build_defined_rescue_iseq, NULL); - rescue = new_child_iseq_with_callback(iseq, ifunc, + rescue = NEW_CHILD_ISEQ_WITH_CALLBACK(ifunc, rb_str_concat(rb_str_new2("defined guard in "), ISEQ_BODY(iseq)->location.label), - iseq, ISEQ_TYPE_RESCUE, 0); + ISEQ_TYPE_RESCUE, 0); lstart->rescued = LABEL_RESCUE_BEG; lend->rescued = LABEL_RESCUE_END; APPEND_LABEL(ret, lcur, lstart); ADD_LABEL(ret, lend); - ADD_CATCH_ENTRY(CATCH_TYPE_RESCUE, lstart, lend, rescue, lfinish[1]); + if (!ignore) { + ADD_CATCH_ENTRY(CATCH_TYPE_RESCUE, lstart, lend, rescue, lfinish[1]); + } } } static int -compile_defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, VALUE needstr) +compile_defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, VALUE needstr, bool ignore) { const int line = nd_line(node); const NODE *line_node = node; @@ -6033,7 +6456,7 @@ compile_defined_expr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const lfinish[0] = NEW_LABEL(line); lfinish[1] = 0; lfinish[2] = 0; - defined_expr(iseq, ret, RNODE_DEFINED(node)->nd_head, lfinish, needstr); + defined_expr(iseq, ret, RNODE_DEFINED(node)->nd_head, lfinish, needstr, ignore); if (lfinish[1]) { ELEM_INSERT_NEXT(last, &new_insn_body(iseq, nd_line(line_node), nd_node_id(line_node), BIN(putnil), 0)->link); ADD_INSN(ret, line_node, swap); @@ -6087,7 +6510,7 @@ add_ensure_range(rb_iseq_t *iseq, struct ensure_range *erange, LABEL *lstart, LABEL *lend) { struct ensure_range *ne = - compile_data_alloc(iseq, sizeof(struct ensure_range)); + compile_data_alloc_type(iseq, struct ensure_range); while (erange->next != 0) { erange = erange->next; @@ -6177,9 +6600,24 @@ keyword_node_single_splat_p(NODE *kwnode) RNODE_LIST(RNODE_LIST(node)->nd_next)->nd_next == NULL; } +static void +compile_single_keyword_splat_mutable(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, + NODE *kwnode, unsigned int *flag_ptr) +{ + *flag_ptr |= VM_CALL_KW_SPLAT_MUT; + ADD_INSN1(args, argn, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + ADD_INSN1(args, argn, newhash, INT2FIX(0)); + compile_hash(iseq, args, kwnode, TRUE, FALSE); + ADD_SEND(args, argn, id_core_hash_merge_kwd, INT2FIX(2)); +} + +#define SPLATARRAY_FALSE 0 +#define SPLATARRAY_TRUE 1 +#define DUP_SINGLE_KW_SPLAT 2 + static int setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, - int dup_rest, unsigned int *flag_ptr, struct rb_callinfo_kwarg **kwarg_ptr) + unsigned int *dup_rest, unsigned int *flag_ptr, struct rb_callinfo_kwarg **kwarg_ptr) { if (!argn) return 0; @@ -6196,7 +6634,12 @@ setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, len -= 1; } else { - compile_hash(iseq, args, kwnode, TRUE, FALSE); + if (keyword_node_single_splat_p(kwnode) && (*dup_rest & DUP_SINGLE_KW_SPLAT)) { + compile_single_keyword_splat_mutable(iseq, args, argn, kwnode, flag_ptr); + } + else { + compile_hash(iseq, args, kwnode, TRUE, FALSE); + } } } @@ -6205,17 +6648,15 @@ setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, case NODE_SPLAT: { // f(*a) NO_CHECK(COMPILE(args, "args (splat)", RNODE_SPLAT(argn)->nd_head)); - ADD_INSN1(args, argn, splatarray, RBOOL(dup_rest)); - if (flag_ptr) { - *flag_ptr |= VM_CALL_ARGS_SPLAT; - if (dup_rest) *flag_ptr |= VM_CALL_ARGS_SPLAT_MUT; - } + ADD_INSN1(args, argn, splatarray, RBOOL(*dup_rest & SPLATARRAY_TRUE)); + if (*dup_rest & SPLATARRAY_TRUE) *dup_rest &= ~SPLATARRAY_TRUE; + if (flag_ptr) *flag_ptr |= VM_CALL_ARGS_SPLAT; RUBY_ASSERT(flag_ptr == NULL || (*flag_ptr & VM_CALL_KW_SPLAT) == 0); return 1; } case NODE_ARGSCAT: { - if (flag_ptr) *flag_ptr |= VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_SPLAT_MUT; - int argc = setup_args_core(iseq, args, RNODE_ARGSCAT(argn)->nd_head, 1, NULL, NULL); + if (flag_ptr) *flag_ptr |= VM_CALL_ARGS_SPLAT; + int argc = setup_args_core(iseq, args, RNODE_ARGSCAT(argn)->nd_head, dup_rest, NULL, NULL); bool args_pushed = false; if (nd_type_p(RNODE_ARGSCAT(argn)->nd_body, NODE_LIST)) { @@ -6230,7 +6671,8 @@ setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, } if (nd_type_p(RNODE_ARGSCAT(argn)->nd_head, NODE_LIST)) { - ADD_INSN1(args, argn, splatarray, Qtrue); + ADD_INSN1(args, argn, splatarray, RBOOL(*dup_rest & SPLATARRAY_TRUE)); + if (*dup_rest & SPLATARRAY_TRUE) *dup_rest &= ~SPLATARRAY_TRUE; argc += 1; } else if (!args_pushed) { @@ -6241,7 +6683,6 @@ setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, if (kwnode) { // kwsplat *flag_ptr |= VM_CALL_KW_SPLAT; - *flag_ptr |= VM_CALL_KW_SPLAT_MUT; compile_hash(iseq, args, kwnode, TRUE, FALSE); argc += 1; } @@ -6249,8 +6690,8 @@ setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, return argc; } case NODE_ARGSPUSH: { - if (flag_ptr) *flag_ptr |= VM_CALL_ARGS_SPLAT | VM_CALL_ARGS_SPLAT_MUT; - int argc = setup_args_core(iseq, args, RNODE_ARGSPUSH(argn)->nd_head, 1, NULL, NULL); + if (flag_ptr) *flag_ptr |= VM_CALL_ARGS_SPLAT; + int argc = setup_args_core(iseq, args, RNODE_ARGSPUSH(argn)->nd_head, dup_rest, NULL, NULL); if (nd_type_p(RNODE_ARGSPUSH(argn)->nd_body, NODE_LIST)) { int rest_len = compile_args(iseq, args, RNODE_ARGSPUSH(argn)->nd_body, &kwnode); @@ -6273,8 +6714,14 @@ setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, *flag_ptr |= VM_CALL_KW_SPLAT; if (!keyword_node_single_splat_p(kwnode)) { *flag_ptr |= VM_CALL_KW_SPLAT_MUT; + compile_hash(iseq, args, kwnode, TRUE, FALSE); + } + else if (*dup_rest & DUP_SINGLE_KW_SPLAT) { + compile_single_keyword_splat_mutable(iseq, args, argn, kwnode, flag_ptr); + } + else { + compile_hash(iseq, args, kwnode, TRUE, FALSE); } - compile_hash(iseq, args, kwnode, TRUE, FALSE); argc += 1; } @@ -6286,18 +6733,148 @@ setup_args_core(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, } } +static void +setup_args_splat_mut(unsigned int *flag, int dup_rest, int initial_dup_rest) +{ + if ((*flag & VM_CALL_ARGS_SPLAT) && dup_rest != initial_dup_rest) { + *flag |= VM_CALL_ARGS_SPLAT_MUT; + } +} + +static bool +setup_args_dup_rest_p(const NODE *argn) +{ + switch(nd_type(argn)) { + case NODE_LVAR: + case NODE_DVAR: + case NODE_GVAR: + case NODE_IVAR: + case NODE_CVAR: + case NODE_CONST: + case NODE_COLON3: + case NODE_INTEGER: + case NODE_FLOAT: + case NODE_RATIONAL: + case NODE_IMAGINARY: + case NODE_STR: + case NODE_SYM: + case NODE_REGX: + case NODE_SELF: + case NODE_NIL: + case NODE_TRUE: + case NODE_FALSE: + case NODE_LAMBDA: + case NODE_NTH_REF: + case NODE_BACK_REF: + return false; + case NODE_COLON2: + return setup_args_dup_rest_p(RNODE_COLON2(argn)->nd_head); + case NODE_LIST: + while (argn) { + if (setup_args_dup_rest_p(RNODE_LIST(argn)->nd_head)) { + return true; + } + argn = RNODE_LIST(argn)->nd_next; + } + return false; + default: + return true; + } +} + static VALUE setup_args(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, unsigned int *flag, struct rb_callinfo_kwarg **keywords) { VALUE ret; + unsigned int dup_rest = SPLATARRAY_TRUE, initial_dup_rest; + + if (argn) { + const NODE *check_arg = nd_type_p(argn, NODE_BLOCK_PASS) ? + RNODE_BLOCK_PASS(argn)->nd_head : argn; + + if (check_arg) { + switch(nd_type(check_arg)) { + case(NODE_SPLAT): + // avoid caller side array allocation for f(*arg) + dup_rest = SPLATARRAY_FALSE; + break; + case(NODE_ARGSCAT): + // avoid caller side array allocation for f(1, *arg) + dup_rest = !nd_type_p(RNODE_ARGSCAT(check_arg)->nd_head, NODE_LIST); + break; + case(NODE_ARGSPUSH): + // avoid caller side array allocation for f(*arg, **hash) and f(1, *arg, **hash) + dup_rest = !((nd_type_p(RNODE_ARGSPUSH(check_arg)->nd_head, NODE_SPLAT) || + (nd_type_p(RNODE_ARGSPUSH(check_arg)->nd_head, NODE_ARGSCAT) && + nd_type_p(RNODE_ARGSCAT(RNODE_ARGSPUSH(check_arg)->nd_head)->nd_head, NODE_LIST))) && + nd_type_p(RNODE_ARGSPUSH(check_arg)->nd_body, NODE_HASH) && + !RNODE_HASH(RNODE_ARGSPUSH(check_arg)->nd_body)->nd_brace); + + if (dup_rest == SPLATARRAY_FALSE) { + // require allocation for keyword key/value/splat that may modify splatted argument + NODE *node = RNODE_HASH(RNODE_ARGSPUSH(check_arg)->nd_body)->nd_head; + while (node) { + NODE *key_node = RNODE_LIST(node)->nd_head; + if (key_node && setup_args_dup_rest_p(key_node)) { + dup_rest = SPLATARRAY_TRUE; + break; + } + + node = RNODE_LIST(node)->nd_next; + NODE *value_node = RNODE_LIST(node)->nd_head; + if (setup_args_dup_rest_p(value_node)) { + dup_rest = SPLATARRAY_TRUE; + break; + } + + node = RNODE_LIST(node)->nd_next; + } + } + break; + default: + break; + } + } + + if (check_arg != argn && setup_args_dup_rest_p(RNODE_BLOCK_PASS(argn)->nd_body)) { + // for block pass that may modify splatted argument, dup rest and kwrest if given + dup_rest = SPLATARRAY_TRUE | DUP_SINGLE_KW_SPLAT; + } + } + initial_dup_rest = dup_rest; + if (argn && nd_type_p(argn, NODE_BLOCK_PASS)) { - unsigned int dup_rest = 1; DECL_ANCHOR(arg_block); INIT_ANCHOR(arg_block); - NO_CHECK(COMPILE(arg_block, "block", RNODE_BLOCK_PASS(argn)->nd_body)); - *flag |= VM_CALL_ARGS_BLOCKARG; + if (RNODE_BLOCK_PASS(argn)->forwarding && ISEQ_BODY(ISEQ_BODY(iseq)->local_iseq)->param.flags.forwardable) { + int idx = ISEQ_BODY(ISEQ_BODY(iseq)->local_iseq)->local_table_size;// - get_local_var_idx(iseq, idDot3); + + RUBY_ASSERT(nd_type_p(RNODE_BLOCK_PASS(argn)->nd_head, NODE_ARGSPUSH)); + const NODE * arg_node = + RNODE_ARGSPUSH(RNODE_BLOCK_PASS(argn)->nd_head)->nd_head; + + int argc = 0; + + // Only compile leading args: + // foo(x, y, ...) + // ^^^^ + if (nd_type_p(arg_node, NODE_ARGSCAT)) { + argc += setup_args_core(iseq, args, RNODE_ARGSCAT(arg_node)->nd_head, &dup_rest, flag, keywords); + } + + *flag |= VM_CALL_FORWARDING; + + ADD_GETLOCAL(args, argn, idx, get_lvar_level(iseq)); + setup_args_splat_mut(flag, dup_rest, initial_dup_rest); + return INT2FIX(argc); + } + else { + *flag |= VM_CALL_ARGS_BLOCKARG; + + NO_CHECK(COMPILE(arg_block, "block", RNODE_BLOCK_PASS(argn)->nd_body)); + } if (LIST_INSN_SIZE_ONE(arg_block)) { LINK_ELEMENT *elem = FIRST_ELEMENT(arg_block); @@ -6306,15 +6883,15 @@ setup_args(rb_iseq_t *iseq, LINK_ANCHOR *const args, const NODE *argn, if (iobj->insn_id == BIN(getblockparam)) { iobj->insn_id = BIN(getblockparamproxy); } - dup_rest = 0; } } - ret = INT2FIX(setup_args_core(iseq, args, RNODE_BLOCK_PASS(argn)->nd_head, dup_rest, flag, keywords)); + ret = INT2FIX(setup_args_core(iseq, args, RNODE_BLOCK_PASS(argn)->nd_head, &dup_rest, flag, keywords)); ADD_SEQ(args, arg_block); } else { - ret = INT2FIX(setup_args_core(iseq, args, argn, 0, flag, keywords)); + ret = INT2FIX(setup_args_core(iseq, args, argn, &dup_rest, flag, keywords)); } + setup_args_splat_mut(flag, dup_rest, initial_dup_rest); return ret; } @@ -6329,7 +6906,7 @@ build_postexe_iseq(rb_iseq_t *iseq, LINK_ANCHOR *ret, const void *ptr) ADD_INSN1(ret, body, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); ADD_CALL_WITH_BLOCK(ret, body, id_core_set_postexe, argc, block); RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)block); - iseq_set_local_table(iseq, 0); + iseq_set_local_table(iseq, 0, 0); } static void @@ -6397,8 +6974,6 @@ optimizable_range_item_p(const NODE *n) { if (!n) return FALSE; switch (nd_type(n)) { - case NODE_LIT: - return RB_INTEGER_TYPE_P(RNODE_LIT(n)->nd_lit); case NODE_LINE: return TRUE; case NODE_INTEGER: @@ -6414,8 +6989,6 @@ static VALUE optimized_range_item(const NODE *n) { switch (nd_type(n)) { - case NODE_LIT: - return RNODE_LIT(n)->nd_lit; case NODE_LINE: return rb_node_line_lineno_val(n); case NODE_INTEGER: @@ -6450,11 +7023,16 @@ compile_if(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int else_label = NEW_LABEL(line); end_label = 0; - compile_branch_condition(iseq, cond_seq, RNODE_IF(node)->nd_cond, then_label, else_label); + NODE *cond = RNODE_IF(node)->nd_cond; + if (nd_type(cond) == NODE_BLOCK) { + cond = RNODE_BLOCK(cond)->nd_head; + } + + CHECK(compile_branch_condition(iseq, cond_seq, cond, then_label, else_label)); ADD_SEQ(ret, cond_seq); if (then_label->refcnt && else_label->refcnt) { - branches = decl_branch_base(iseq, node, type == NODE_IF ? "if" : "unless"); + branches = decl_branch_base(iseq, PTR2NUM(node), nd_code_loc(node), type == NODE_IF ? "if" : "unless"); } if (then_label->refcnt) { @@ -6465,10 +7043,12 @@ compile_if(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int CHECK(COMPILE_(then_seq, "then", node_body, popped)); if (else_label->refcnt) { + const NODE *const coverage_node = node_body ? node_body : node; add_trace_branch_coverage( iseq, ret, - node_body ? node_body : node, + nd_code_loc(coverage_node), + nd_node_id(coverage_node), 0, type == NODE_IF ? "then" : "else", branches); @@ -6489,10 +7069,12 @@ compile_if(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int CHECK(COMPILE_(else_seq, "else", node_else, popped)); if (then_label->refcnt) { + const NODE *const coverage_node = node_else ? node_else : node; add_trace_branch_coverage( iseq, ret, - node_else ? node_else : node, + nd_code_loc(coverage_node), + nd_node_id(coverage_node), 1, type == NODE_IF ? "else" : "then", branches); @@ -6517,7 +7099,7 @@ compile_case(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_nod DECL_ANCHOR(body_seq); DECL_ANCHOR(cond_seq); int only_special_literals = 1; - VALUE literals = rb_hash_new(); + VALUE literals = cdhash_new(0); int line; enum node_type type; const NODE *line_node; @@ -6528,11 +7110,9 @@ compile_case(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_nod INIT_ANCHOR(body_seq); INIT_ANCHOR(cond_seq); - RHASH_TBL_RAW(literals)->type = &cdhash_type; - CHECK(COMPILE(head, "case base", RNODE_CASE(node)->nd_head)); - branches = decl_branch_base(iseq, node, "case"); + branches = decl_branch_base(iseq, PTR2NUM(node), nd_code_loc(node), "case"); node = RNODE_CASE(node)->nd_body; EXPECT_NODE("NODE_CASE", node, NODE_WHEN, COMPILE_NG); @@ -6551,13 +7131,17 @@ compile_case(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_nod l1 = NEW_LABEL(line); ADD_LABEL(body_seq, l1); ADD_INSN(body_seq, line_node, pop); + + const NODE *const coverage_node = RNODE_WHEN(node)->nd_body ? RNODE_WHEN(node)->nd_body : node; add_trace_branch_coverage( iseq, body_seq, - RNODE_WHEN(node)->nd_body ? RNODE_WHEN(node)->nd_body : node, + nd_code_loc(coverage_node), + nd_node_id(coverage_node), branch_id++, "when", branches); + CHECK(COMPILE_(body_seq, "when body", RNODE_WHEN(node)->nd_body, popped)); ADD_INSNL(body_seq, line_node, jump, endlabel); @@ -6594,7 +7178,7 @@ compile_case(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_nod if (node) { ADD_LABEL(cond_seq, elselabel); ADD_INSN(cond_seq, line_node, pop); - add_trace_branch_coverage(iseq, cond_seq, node, branch_id, "else", branches); + add_trace_branch_coverage(iseq, cond_seq, nd_code_loc(node), nd_node_id(node), branch_id, "else", branches); CHECK(COMPILE_(cond_seq, "else", node, popped)); ADD_INSNL(cond_seq, line_node, jump, endlabel); } @@ -6602,7 +7186,7 @@ compile_case(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_nod debugs("== else (implicit)\n"); ADD_LABEL(cond_seq, elselabel); ADD_INSN(cond_seq, orig_node, pop); - add_trace_branch_coverage(iseq, cond_seq, orig_node, branch_id, "else", branches); + add_trace_branch_coverage(iseq, cond_seq, nd_code_loc(orig_node), nd_node_id(orig_node), branch_id, "else", branches); if (!popped) { ADD_INSN(cond_seq, orig_node, putnil); } @@ -6633,7 +7217,7 @@ compile_case2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_no VALUE branches = Qfalse; int branch_id = 0; - branches = decl_branch_base(iseq, orig_node, "case"); + branches = decl_branch_base(iseq, PTR2NUM(orig_node), nd_code_loc(orig_node), "case"); INIT_ANCHOR(body_seq); endlabel = NEW_LABEL(nd_line(node)); @@ -6642,13 +7226,17 @@ compile_case2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_no const int line = nd_line(node); LABEL *l1 = NEW_LABEL(line); ADD_LABEL(body_seq, l1); + + const NODE *const coverage_node = RNODE_WHEN(node)->nd_body ? RNODE_WHEN(node)->nd_body : node; add_trace_branch_coverage( iseq, body_seq, - RNODE_WHEN(node)->nd_body ? RNODE_WHEN(node)->nd_body : node, + nd_code_loc(coverage_node), + nd_node_id(coverage_node), branch_id++, "when", branches); + CHECK(COMPILE_(body_seq, "when", RNODE_WHEN(node)->nd_body, popped)); ADD_INSNL(body_seq, node, jump, endlabel); @@ -6682,10 +7270,12 @@ compile_case2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_no node = RNODE_WHEN(node)->nd_next; } /* else */ + const NODE *const coverage_node = node ? node : orig_node; add_trace_branch_coverage( iseq, ret, - node ? node : orig_node, + nd_code_loc(coverage_node), + nd_node_id(coverage_node), branch_id, "else", branches); @@ -7140,6 +7730,7 @@ iseq_compile_pattern_each(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *c ADD_INSN(ret, line_node, putnil); } else { + RB_OBJ_SET_FROZEN_SHAREABLE(keys); ADD_INSN1(ret, line_node, duparray, keys); RB_OBJ_WRITTEN(iseq, Qundef, rb_obj_hide(keys)); } @@ -7177,7 +7768,8 @@ iseq_compile_pattern_each(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *c ADD_INSN(ret, line_node, dup); ADD_INSNL(ret, line_node, branchif, match_succeeded); - ADD_INSN1(ret, line_node, putobject, rb_str_freeze(rb_sprintf("key not found: %+"PRIsVALUE, key))); // (4) + VALUE str = rb_str_freeze(rb_sprintf("key not found: %+"PRIsVALUE, key)); + ADD_INSN1(ret, line_node, putobject, RB_OBJ_SET_SHAREABLE(str)); // (4) ADD_INSN1(ret, line_node, setn, INT2FIX(base_index + CASE3_BI_OFFSET_ERROR_STRING + 2 /* (3), (4) */)); ADD_INSN1(ret, line_node, putobject, Qtrue); // (5) ADD_INSN1(ret, line_node, setn, INT2FIX(base_index + CASE3_BI_OFFSET_KEY_ERROR_P + 3 /* (3), (4), (5) */)); @@ -7241,7 +7833,6 @@ iseq_compile_pattern_each(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *c ADD_INSNL(ret, line_node, jump, unmatched); break; } - case NODE_LIT: case NODE_SYM: case NODE_REGX: case NODE_LINE: @@ -7625,7 +8216,7 @@ compile_case3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_no INIT_ANCHOR(body_seq); INIT_ANCHOR(cond_seq); - branches = decl_branch_base(iseq, node, "case"); + branches = decl_branch_base(iseq, PTR2NUM(node), nd_code_loc(node), "case"); node = RNODE_CASE3(node)->nd_body; EXPECT_NODE("NODE_CASE3", node, NODE_IN, COMPILE_NG); @@ -7659,13 +8250,17 @@ compile_case3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_no l1 = NEW_LABEL(line); ADD_LABEL(body_seq, l1); ADD_INSN1(body_seq, line_node, adjuststack, INT2FIX(single_pattern ? 6 : 2)); + + const NODE *const coverage_node = RNODE_IN(node)->nd_body ? RNODE_IN(node)->nd_body : node; add_trace_branch_coverage( iseq, body_seq, - RNODE_IN(node)->nd_body ? RNODE_IN(node)->nd_body : node, + nd_code_loc(coverage_node), + nd_node_id(coverage_node), branch_id++, "in", branches); + CHECK(COMPILE_(body_seq, "in body", RNODE_IN(node)->nd_body, popped)); ADD_INSNL(body_seq, line_node, jump, endlabel); @@ -7697,7 +8292,7 @@ compile_case3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_no ADD_LABEL(cond_seq, elselabel); ADD_INSN(cond_seq, line_node, pop); ADD_INSN(cond_seq, line_node, pop); /* discard cached #deconstruct value */ - add_trace_branch_coverage(iseq, cond_seq, node, branch_id, "else", branches); + add_trace_branch_coverage(iseq, cond_seq, nd_code_loc(node), nd_node_id(node), branch_id, "else", branches); CHECK(COMPILE_(cond_seq, "else", node, popped)); ADD_INSNL(cond_seq, line_node, jump, endlabel); ADD_INSN(cond_seq, line_node, putnil); @@ -7708,7 +8303,7 @@ compile_case3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const orig_no else { debugs("== else (implicit)\n"); ADD_LABEL(cond_seq, elselabel); - add_trace_branch_coverage(iseq, cond_seq, orig_node, branch_id, "else", branches); + add_trace_branch_coverage(iseq, cond_seq, nd_code_loc(orig_node), nd_node_id(orig_node), branch_id, "else", branches); ADD_INSN1(cond_seq, orig_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); if (single_pattern) { @@ -7825,25 +8420,29 @@ compile_loop(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, in if (tmp_label) ADD_LABEL(ret, tmp_label); ADD_LABEL(ret, redo_label); - branches = decl_branch_base(iseq, node, type == NODE_WHILE ? "while" : "until"); + branches = decl_branch_base(iseq, PTR2NUM(node), nd_code_loc(node), type == NODE_WHILE ? "while" : "until"); + + const NODE *const coverage_node = RNODE_WHILE(node)->nd_body ? RNODE_WHILE(node)->nd_body : node; add_trace_branch_coverage( iseq, ret, - RNODE_WHILE(node)->nd_body ? RNODE_WHILE(node)->nd_body : node, + nd_code_loc(coverage_node), + nd_node_id(coverage_node), 0, "body", branches); + CHECK(COMPILE_POPPED(ret, "while body", RNODE_WHILE(node)->nd_body)); ADD_LABEL(ret, next_label); /* next */ if (type == NODE_WHILE) { - compile_branch_condition(iseq, ret, RNODE_WHILE(node)->nd_cond, - redo_label, end_label); + CHECK(compile_branch_condition(iseq, ret, RNODE_WHILE(node)->nd_cond, + redo_label, end_label)); } else { /* until */ - compile_branch_condition(iseq, ret, RNODE_WHILE(node)->nd_cond, - end_label, redo_label); + CHECK(compile_branch_condition(iseq, ret, RNODE_WHILE(node)->nd_cond, + end_label, redo_label)); } ADD_LABEL(ret, end_label); @@ -7913,11 +8512,11 @@ compile_iter(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, in // Normally, "send" instruction is at the last. // However, qcall under branch coverage measurement adds some instructions after the "send". // - // Note that "invokesuper" appears instead of "send". + // Note that "invokesuper", "invokesuperforward" appears instead of "send". INSN *iobj; LINK_ELEMENT *last_elem = LAST_ELEMENT(ret); iobj = IS_INSN(last_elem) ? (INSN*) last_elem : (INSN*) get_prev_insn((INSN*) last_elem); - while (INSN_OF(iobj) != BIN(send) && INSN_OF(iobj) != BIN(invokesuper)) { + while (!IS_INSN_ID(iobj, send) && !IS_INSN_ID(iobj, invokesuper) && !IS_INSN_ID(iobj, sendforward) && !IS_INSN_ID(iobj, invokesuperforward)) { iobj = (INSN*) get_prev_insn(iobj); } ELEM_INSERT_NEXT(&iobj->link, (LINK_ELEMENT*) retry_end_l); @@ -8057,7 +8656,6 @@ compile_next(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, in add_ensure_iseq(ret, iseq, 0); ADD_INSNL(ret, line_node, jump, ISEQ_COMPILE_DATA(iseq)->end_label); ADD_ADJUST_RESTORE(ret, splabel); - splabel->unremovable = FALSE; if (!popped) { ADD_INSN(ret, line_node, putnil); @@ -8281,7 +8879,11 @@ compile_resbody(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, ADD_LABEL(ret, label_hit); ADD_TRACE(ret, RUBY_EVENT_RESCUE); - if (nd_type(RNODE_RESBODY(resq)->nd_body) == NODE_BEGIN && RNODE_BEGIN(RNODE_RESBODY(resq)->nd_body)->nd_body == NULL) { + if (RNODE_RESBODY(resq)->nd_exc_var) { + CHECK(COMPILE_POPPED(ret, "resbody exc_var", RNODE_RESBODY(resq)->nd_exc_var)); + } + + if (nd_type(RNODE_RESBODY(resq)->nd_body) == NODE_BEGIN && RNODE_BEGIN(RNODE_RESBODY(resq)->nd_body)->nd_body == NULL && !RNODE_RESBODY(resq)->nd_exc_var) { // empty body ADD_SYNTHETIC_INSN(ret, nd_line(RNODE_RESBODY(resq)->nd_body), -1, putnil); } @@ -8302,7 +8904,7 @@ compile_resbody(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, static int compile_ensure(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { - const int line = nd_line(node); + const int line = nd_line(RNODE_ENSURE(node)->nd_ensr); const NODE *line_node = node; DECL_ANCHOR(ensr); const rb_iseq_t *ensure = NEW_CHILD_ISEQ(RNODE_ENSURE(node)->nd_ensr, @@ -8407,6 +9009,27 @@ compile_return(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, return COMPILE_OK; } +static bool +drop_unreachable_return(LINK_ANCHOR *ret) +{ + LINK_ELEMENT *i = ret->last, *last; + if (!i) return false; + if (IS_TRACE(i)) i = i->prev; + if (!IS_INSN(i) || !IS_INSN_ID(i, putnil)) return false; + last = i = i->prev; + if (IS_ADJUST(i)) i = i->prev; + if (!IS_INSN(i)) return false; + switch (INSN_OF(i)) { + case BIN(leave): + case BIN(jump): + break; + default: + return false; + } + (ret->last = last->prev)->next = NULL; + return true; +} + static int compile_evstr(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { @@ -8440,11 +9063,11 @@ qcall_branch_start(rb_iseq_t *iseq, LINK_ANCHOR *const recv, VALUE *branches, co LABEL *else_label = NEW_LABEL(nd_line(line_node)); VALUE br = 0; - br = decl_branch_base(iseq, node, "&."); + br = decl_branch_base(iseq, PTR2NUM(node), nd_code_loc(node), "&."); *branches = br; ADD_INSN(recv, line_node, dup); ADD_INSNL(recv, line_node, branchnil, else_label); - add_trace_branch_coverage(iseq, recv, node, 0, "then", br); + add_trace_branch_coverage(iseq, recv, nd_code_loc(node), nd_node_id(node), 0, "then", br); return else_label; } @@ -8456,7 +9079,7 @@ qcall_branch_end(rb_iseq_t *iseq, LINK_ANCHOR *const ret, LABEL *else_label, VAL end_label = NEW_LABEL(nd_line(line_node)); ADD_INSNL(ret, line_node, jump, end_label); ADD_LABEL(ret, else_label); - add_trace_branch_coverage(iseq, ret, node, 1, "else", branches); + add_trace_branch_coverage(iseq, ret, nd_code_loc(node), nd_node_id(node), 1, "else", branches); ADD_LABEL(ret, end_label); } @@ -8472,7 +9095,7 @@ compile_call_precheck_freeze(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE get_nd_args(node) == NULL && ISEQ_COMPILE_DATA(iseq)->current_block == NULL && ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) { - VALUE str = rb_fstring(get_string_value(get_nd_recv(node))); + VALUE str = get_string_value(get_nd_recv(node)); if (get_node_call_nd_mid(node) == idUMinus) { ADD_INSN2(ret, line_node, opt_str_uminus, str, new_callinfo(iseq, idUMinus, 0, 0, NULL, FALSE)); @@ -8487,25 +9110,6 @@ compile_call_precheck_freeze(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE } return TRUE; } - /* optimization shortcut - * obj["literal"] -> opt_aref_with(obj, "literal") - */ - if (get_node_call_nd_mid(node) == idAREF && !private_recv_p(node) && get_nd_args(node) && - nd_type_p(get_nd_args(node), NODE_LIST) && RNODE_LIST(get_nd_args(node))->as.nd_alen == 1 && - (nd_type_p(RNODE_LIST(get_nd_args(node))->nd_head, NODE_STR) || nd_type_p(RNODE_LIST(get_nd_args(node))->nd_head, NODE_FILE)) && - ISEQ_COMPILE_DATA(iseq)->current_block == NULL && - !frozen_string_literal_p(iseq) && - ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) { - VALUE str = rb_fstring(get_string_value(RNODE_LIST(get_nd_args(node))->nd_head)); - CHECK(COMPILE(ret, "recv", get_nd_recv(node))); - ADD_INSN2(ret, line_node, opt_aref_with, str, - new_callinfo(iseq, idAREF, 1, 0, NULL, FALSE)); - RB_OBJ_WRITTEN(iseq, Qundef, str); - if (popped) { - ADD_INSN(ret, line_node, pop); - } - return TRUE; - } return FALSE; } @@ -8643,22 +9247,29 @@ compile_builtin_attr(rb_iseq_t *iseq, const NODE *node) case NODE_SYM: symbol = rb_node_sym_string_val(node); break; - case NODE_LIT: - symbol = RNODE_LIT(node)->nd_lit; - break; default: goto bad_arg; } if (!SYMBOL_P(symbol)) goto non_symbol_arg; - string = rb_sym_to_s(symbol); + string = rb_sym2str(symbol); if (strcmp(RSTRING_PTR(string), "leaf") == 0) { ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_LEAF; } else if (strcmp(RSTRING_PTR(string), "inline_block") == 0) { ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_INLINE_BLOCK; } + else if (strcmp(RSTRING_PTR(string), "use_block") == 0) { + iseq_set_use_block(iseq); + } + else if (strcmp(RSTRING_PTR(string), "c_trace") == 0) { + // Let the iseq act like a C method in backtraces + ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_C_TRACE; + } + else if (strcmp(RSTRING_PTR(string), "without_interrupts") == 0) { + ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_WITHOUT_INTERRUPTS; + } else { goto unknown_arg; } @@ -8692,9 +9303,6 @@ compile_builtin_arg(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, c case NODE_SYM: name = rb_node_sym_string_val(node); break; - case NODE_LIT: - name = RNODE_LIT(node)->nd_lit; - break; default: goto bad_arg; } @@ -8763,20 +9371,18 @@ compile_builtin_mandatory_only_method(rb_iseq_t *iseq, const NODE *node, const N rb_node_init(RNODE(&scope_node), NODE_SCOPE); scope_node.nd_tbl = tbl; scope_node.nd_body = mandatory_node(iseq, node); + scope_node.nd_parent = NULL; scope_node.nd_args = &args_node; - rb_ast_body_t ast = { - .root = RNODE(&scope_node), - .frozen_string_literal = -1, - .coverage_enabled = -1, - .script_lines = ISEQ_BODY(iseq)->variable.script_lines, - }; + VALUE ast_value = rb_ruby_ast_new(RNODE(&scope_node)); - ISEQ_BODY(iseq)->mandatory_only_iseq = - rb_iseq_new_with_opt(&ast, rb_iseq_base_label(iseq), + const rb_iseq_t *mandatory_only_iseq = + rb_iseq_new_with_opt(ast_value, rb_iseq_base_label(iseq), rb_iseq_path(iseq), rb_iseq_realpath(iseq), nd_line(line_node), NULL, 0, - ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option); + ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option, + ISEQ_BODY(iseq)->variable.script_lines); + RB_OBJ_WRITE(iseq, &ISEQ_BODY(iseq)->mandatory_only_iseq, (VALUE)mandatory_only_iseq); ALLOCV_END(idtmp); return COMPILE_OK; @@ -8845,7 +9451,7 @@ compile_builtin_function_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NOD if (cconst) { typedef VALUE(*builtin_func0)(void *, VALUE); - VALUE const_val = (*(builtin_func0)bf->func_ptr)(NULL, Qnil); + VALUE const_val = (*(builtin_func0)(uintptr_t)bf->func_ptr)(NULL, Qnil); ADD_INSN1(ret, line_node, putobject, const_val); return COMPILE_OK; } @@ -8897,6 +9503,7 @@ compile_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, co INIT_ANCHOR(recv); INIT_ANCHOR(args); + #if OPT_SUPPORT_JOKE if (nd_type_p(node, NODE_VCALL)) { ID id_bitblt; @@ -8933,20 +9540,7 @@ compile_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, co labels_table = st_init_numtable(); ISEQ_COMPILE_DATA(iseq)->labels_table = labels_table; } - if (nd_type_p(node->nd_args->nd_head, NODE_LIT) && - SYMBOL_P(node->nd_args->nd_head->nd_lit)) { - - label_name = node->nd_args->nd_head->nd_lit; - if (!st_lookup(labels_table, (st_data_t)label_name, &data)) { - label = NEW_LABEL(nd_line(line_node)); - label->position = nd_line(line_node); - st_insert(labels_table, (st_data_t)label_name, (st_data_t)label); - } - else { - label = (LABEL *)data; - } - } - else { + { COMPILE_ERROR(ERROR_ARGS "invalid goto/label format"); return COMPILE_NG; } @@ -9005,6 +9599,17 @@ compile_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, co } ADD_SEQ(ret, recv); + + bool inline_new = ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction && + mid == rb_intern("new") && + parent_block == NULL && + !(flag & VM_CALL_ARGS_BLOCKARG); + + if (inline_new) { + ADD_INSN(ret, node, putnil); + ADD_INSN(ret, node, swap); + } + ADD_SEQ(ret, args); debugp_param("call args argc", argc); @@ -9021,7 +9626,37 @@ compile_call(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, co if ((flag & VM_CALL_ARGS_BLOCKARG) && (flag & VM_CALL_KW_SPLAT) && !(flag & VM_CALL_KW_SPLAT_MUT)) { ADD_INSN(ret, line_node, splatkw); } - ADD_SEND_R(ret, line_node, mid, argc, parent_block, INT2FIX(flag), keywords); + + LABEL *not_basic_new = NEW_LABEL(nd_line(node)); + LABEL *not_basic_new_finish = NEW_LABEL(nd_line(node)); + + if (inline_new) { + // Jump unless the receiver uses the "basic" implementation of "new" + VALUE ci; + if (flag & VM_CALL_FORWARDING) { + ci = (VALUE)new_callinfo(iseq, mid, NUM2INT(argc) + 1, flag, keywords, 0); + } + else { + ci = (VALUE)new_callinfo(iseq, mid, NUM2INT(argc), flag, keywords, 0); + } + ADD_INSN2(ret, node, opt_new, ci, not_basic_new); + LABEL_REF(not_basic_new); + + // optimized path + ADD_SEND_R(ret, line_node, rb_intern("initialize"), argc, parent_block, INT2FIX(flag | VM_CALL_FCALL), keywords); + ADD_INSNL(ret, line_node, jump, not_basic_new_finish); + + ADD_LABEL(ret, not_basic_new); + // Fall back to normal send + ADD_SEND_R(ret, line_node, mid, argc, parent_block, INT2FIX(flag), keywords); + ADD_INSN(ret, line_node, swap); + + ADD_LABEL(ret, not_basic_new_finish); + ADD_INSN(ret, line_node, pop); + } + else { + ADD_SEND_R(ret, line_node, mid, argc, parent_block, INT2FIX(flag), keywords); + } qcall_branch_end(iseq, ret, else_label, branches, node, line_node); if (popped) { @@ -9038,9 +9673,6 @@ compile_op_asgn1(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node unsigned int flag = 0; int asgnflag = 0; ID id = RNODE_OP_ASGN1(node)->nd_mid; - int boff = 0; - int keyword_len = 0; - struct rb_callinfo_kwarg *keywords = NULL; /* * a[x] (op)= y @@ -9074,32 +9706,14 @@ compile_op_asgn1(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node case NODE_ZLIST: argc = INT2FIX(0); break; - case NODE_BLOCK_PASS: - boff = 1; - /* fall through */ default: - argc = setup_args(iseq, ret, RNODE_OP_ASGN1(node)->nd_index, &flag, &keywords); - if (flag & VM_CALL_KW_SPLAT) { - if (boff) { - ADD_INSN(ret, node, splatkw); - } - else { - /* Make sure to_hash is only called once and not twice */ - ADD_INSN(ret, node, dup); - ADD_INSN(ret, node, splatkw); - ADD_INSN(ret, node, pop); - } - } + argc = setup_args(iseq, ret, RNODE_OP_ASGN1(node)->nd_index, &flag, NULL); CHECK(!NIL_P(argc)); } - int dup_argn = FIX2INT(argc) + 1 + boff; - if (keywords) { - keyword_len = keywords->keyword_len; - dup_argn += keyword_len; - } + int dup_argn = FIX2INT(argc) + 1; ADD_INSN1(ret, node, dupn, INT2FIX(dup_argn)); flag |= asgnflag; - ADD_SEND_R(ret, node, idAREF, argc, NULL, INT2FIX(flag & ~(VM_CALL_ARGS_SPLAT_MUT|VM_CALL_KW_SPLAT_MUT)), keywords); + ADD_SEND_R(ret, node, idAREF, argc, NULL, INT2FIX(flag & ~VM_CALL_ARGS_SPLAT_MUT), NULL); if (id == idOROP || id == idANDOP) { /* a[x] ||= y or a[x] &&= y @@ -9127,57 +9741,17 @@ compile_op_asgn1(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node ADD_INSN1(ret, node, setn, INT2FIX(dup_argn+1)); } if (flag & VM_CALL_ARGS_SPLAT) { - if (flag & VM_CALL_KW_SPLAT) { - ADD_INSN1(ret, node, topn, INT2FIX(2 + boff)); - if (!(flag & VM_CALL_ARGS_SPLAT_MUT)) { - ADD_INSN1(ret, node, splatarray, Qtrue); - flag |= VM_CALL_ARGS_SPLAT_MUT; - } + if (!(flag & VM_CALL_ARGS_SPLAT_MUT)) { ADD_INSN(ret, node, swap); - ADD_INSN1(ret, node, pushtoarray, INT2FIX(1)); - ADD_INSN1(ret, node, setn, INT2FIX(2 + boff)); - ADD_INSN(ret, node, pop); - } - else { - if (boff > 0) { - ADD_INSN1(ret, node, dupn, INT2FIX(3)); - ADD_INSN(ret, node, swap); - ADD_INSN(ret, node, pop); - } - if (!(flag & VM_CALL_ARGS_SPLAT_MUT)) { - ADD_INSN(ret, node, swap); - ADD_INSN1(ret, node, splatarray, Qtrue); - ADD_INSN(ret, node, swap); - flag |= VM_CALL_ARGS_SPLAT_MUT; - } - ADD_INSN1(ret, node, pushtoarray, INT2FIX(1)); - if (boff > 0) { - ADD_INSN1(ret, node, setn, INT2FIX(3)); - ADD_INSN(ret, node, pop); - ADD_INSN(ret, node, pop); - } - } - ADD_SEND_R(ret, node, idASET, argc, NULL, INT2FIX(flag), keywords); - } - else if (flag & VM_CALL_KW_SPLAT) { - if (boff > 0) { - ADD_INSN1(ret, node, topn, INT2FIX(2)); + ADD_INSN1(ret, node, splatarray, Qtrue); ADD_INSN(ret, node, swap); - ADD_INSN1(ret, node, setn, INT2FIX(3)); - ADD_INSN(ret, node, pop); + flag |= VM_CALL_ARGS_SPLAT_MUT; } - ADD_INSN(ret, node, swap); - ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), keywords); - } - else if (keyword_len) { - ADD_INSN1(ret, node, opt_reverse, INT2FIX(keyword_len+boff+1)); - ADD_INSN1(ret, node, opt_reverse, INT2FIX(keyword_len+boff+0)); - ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), keywords); + ADD_INSN1(ret, node, pushtoarray, INT2FIX(1)); + ADD_SEND_R(ret, node, idASET, argc, NULL, INT2FIX(flag), NULL); } else { - if (boff > 0) - ADD_INSN(ret, node, swap); - ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), keywords); + ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), NULL); } ADD_INSN(ret, node, pop); ADD_INSNL(ret, node, jump, lfin); @@ -9196,22 +9770,17 @@ compile_op_asgn1(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node } if (flag & VM_CALL_ARGS_SPLAT) { if (flag & VM_CALL_KW_SPLAT) { - ADD_INSN1(ret, node, topn, INT2FIX(2 + boff)); + ADD_INSN1(ret, node, topn, INT2FIX(2)); if (!(flag & VM_CALL_ARGS_SPLAT_MUT)) { ADD_INSN1(ret, node, splatarray, Qtrue); flag |= VM_CALL_ARGS_SPLAT_MUT; } ADD_INSN(ret, node, swap); ADD_INSN1(ret, node, pushtoarray, INT2FIX(1)); - ADD_INSN1(ret, node, setn, INT2FIX(2 + boff)); + ADD_INSN1(ret, node, setn, INT2FIX(2)); ADD_INSN(ret, node, pop); } else { - if (boff > 0) { - ADD_INSN1(ret, node, dupn, INT2FIX(3)); - ADD_INSN(ret, node, swap); - ADD_INSN(ret, node, pop); - } if (!(flag & VM_CALL_ARGS_SPLAT_MUT)) { ADD_INSN(ret, node, swap); ADD_INSN1(ret, node, splatarray, Qtrue); @@ -9219,35 +9788,11 @@ compile_op_asgn1(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node flag |= VM_CALL_ARGS_SPLAT_MUT; } ADD_INSN1(ret, node, pushtoarray, INT2FIX(1)); - if (boff > 0) { - ADD_INSN1(ret, node, setn, INT2FIX(3)); - ADD_INSN(ret, node, pop); - ADD_INSN(ret, node, pop); - } } - ADD_SEND_R(ret, node, idASET, argc, NULL, INT2FIX(flag), keywords); - } - else if (flag & VM_CALL_KW_SPLAT) { - if (boff > 0) { - ADD_INSN1(ret, node, topn, INT2FIX(2)); - ADD_INSN(ret, node, swap); - ADD_INSN1(ret, node, setn, INT2FIX(3)); - ADD_INSN(ret, node, pop); - } - ADD_INSN(ret, node, swap); - ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), keywords); - } - else if (keyword_len) { - ADD_INSN(ret, node, dup); - ADD_INSN1(ret, node, opt_reverse, INT2FIX(keyword_len+boff+2)); - ADD_INSN1(ret, node, opt_reverse, INT2FIX(keyword_len+boff+1)); - ADD_INSN(ret, node, pop); - ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), keywords); + ADD_SEND_R(ret, node, idASET, argc, NULL, INT2FIX(flag), NULL); } else { - if (boff > 0) - ADD_INSN(ret, node, swap); - ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), keywords); + ADD_SEND_R(ret, node, idASET, FIXNUM_INC(argc, 1), NULL, INT2FIX(flag), NULL); } ADD_INSN(ret, node, pop); } @@ -9374,6 +9919,8 @@ compile_op_asgn2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node return COMPILE_OK; } +static int compile_shareable_constant_value(rb_iseq_t *iseq, LINK_ANCHOR *ret, enum rb_parser_shareability shareable, const NODE *lhs, const NODE *value); + static int compile_op_cdecl(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped) { @@ -9417,7 +9964,7 @@ compile_op_cdecl(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node /* cref [obj] */ if (!popped) ADD_INSN(ret, node, pop); /* cref */ if (lassign) ADD_LABEL(ret, lassign); - CHECK(COMPILE(ret, "NODE_OP_CDECL#nd_value", RNODE_OP_CDECL(node)->nd_value)); + CHECK(compile_shareable_constant_value(iseq, ret, RNODE_OP_CDECL(node)->shareability, RNODE_OP_CDECL(node)->nd_head, RNODE_OP_CDECL(node)->nd_value)); /* cref value */ if (popped) ADD_INSN1(ret, node, topn, INT2FIX(1)); /* cref value cref */ @@ -9431,7 +9978,7 @@ compile_op_cdecl(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node ADD_INSN(ret, node, pop); /* [value] */ } else { - CHECK(COMPILE(ret, "NODE_OP_CDECL#nd_value", RNODE_OP_CDECL(node)->nd_value)); + CHECK(compile_shareable_constant_value(iseq, ret, RNODE_OP_CDECL(node)->shareability, RNODE_OP_CDECL(node)->nd_head, RNODE_OP_CDECL(node)->nd_value)); /* cref obj value */ ADD_CALL(ret, node, RNODE_OP_CDECL(node)->nd_aid, INT2FIX(1)); /* cref value */ @@ -9456,7 +10003,7 @@ compile_op_log(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, LABEL *lfinish[2]; lfinish[0] = lfin; lfinish[1] = 0; - defined_expr(iseq, ret, RNODE_OP_ASGN_OR(node)->nd_head, lfinish, Qfalse); + defined_expr(iseq, ret, RNODE_OP_ASGN_OR(node)->nd_head, lfinish, Qfalse, false); lassign = lfinish[1]; if (!lassign) { lassign = NEW_LABEL(line); @@ -9499,9 +10046,11 @@ compile_super(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i unsigned int flag = 0; struct rb_callinfo_kwarg *keywords = NULL; const rb_iseq_t *parent_block = ISEQ_COMPILE_DATA(iseq)->current_block; + int use_block = 1; INIT_ANCHOR(args); ISEQ_COMPILE_DATA(iseq)->current_block = NULL; + if (type == NODE_SUPER) { VALUE vargc = setup_args(iseq, args, RNODE_SUPER(node)->nd_args, &flag, &keywords); CHECK(!NIL_P(vargc)); @@ -9509,6 +10058,10 @@ compile_super(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i if ((flag & VM_CALL_ARGS_BLOCKARG) && (flag & VM_CALL_KW_SPLAT) && !(flag & VM_CALL_KW_SPLAT_MUT)) { ADD_INSN(args, node, splatkw); } + + if (flag & VM_CALL_ARGS_BLOCKARG) { + use_block = 0; + } } else { /* NODE_ZSUPER */ @@ -9526,6 +10079,13 @@ compile_super(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i ADD_GETLOCAL(args, node, idx, lvar_level); } + /* forward ... */ + if (local_body->param.flags.forwardable) { + flag |= VM_CALL_FORWARDING; + int idx = local_body->local_table_size - get_local_var_idx(liseq, idDot3); + ADD_GETLOCAL(args, node, idx, lvar_level); + } + if (local_body->param.flags.has_opt) { /* optional arguments */ int j; @@ -9602,13 +10162,23 @@ compile_super(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i } } + if (use_block && parent_block == NULL) { + iseq_set_use_block(ISEQ_BODY(iseq)->local_iseq); + } + flag |= VM_CALL_SUPER | VM_CALL_FCALL; if (type == NODE_ZSUPER) flag |= VM_CALL_ZSUPER; ADD_INSN(ret, node, putself); ADD_SEQ(ret, args); - ADD_INSN2(ret, node, invokesuper, - new_callinfo(iseq, 0, argc, flag, keywords, parent_block != NULL), - parent_block); + + const struct rb_callinfo * ci = new_callinfo(iseq, 0, argc, flag, keywords, parent_block != NULL); + + if (vm_ci_flag(ci) & VM_CALL_FORWARDING) { + ADD_INSN2(ret, node, invokesuperforward, ci, parent_block); + } + else { + ADD_INSN2(ret, node, invokesuper, ci, parent_block); + } if (popped) { ADD_INSN(ret, node, pop); @@ -9645,6 +10215,7 @@ compile_yield(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i ADD_SEQ(ret, args); ADD_INSN1(ret, node, invokeblock, new_callinfo(iseq, 0, FIX2INT(argc), flag, keywords, FALSE)); + iseq_set_use_block(ISEQ_BODY(iseq)->local_iseq); if (popped) { ADD_INSN(ret, node, pop); @@ -9670,9 +10241,13 @@ compile_match(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, i INIT_ANCHOR(val); switch ((int)type) { case NODE_MATCH: - ADD_INSN1(recv, node, putobject, rb_node_regx_string_val(node)); - ADD_INSN2(val, node, getspecial, INT2FIX(0), - INT2FIX(0)); + { + VALUE re = rb_node_regx_string_val(node); + RB_OBJ_SET_FROZEN_SHAREABLE(re); + ADD_INSN1(recv, node, putobject, re); + ADD_INSN2(val, node, getspecial, INT2FIX(0), + INT2FIX(0)); + } break; case NODE_MATCH2: CHECK(COMPILE(recv, "receiver", RNODE_MATCH2(node)->nd_recv)); @@ -9749,6 +10324,7 @@ compile_colon3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { ISEQ_BODY(iseq)->ic_size++; VALUE segments = rb_ary_new_from_args(2, ID2SYM(idNULL), ID2SYM(RNODE_COLON3(node)->nd_mid)); + RB_OBJ_SET_FROZEN_SHAREABLE(segments); ADD_INSN1(ret, node, opt_getconstant_path, segments); RB_OBJ_WRITTEN(iseq, Qundef, segments); } @@ -9776,6 +10352,7 @@ compile_dots(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, in VALUE bv = optimized_range_item(b); VALUE ev = optimized_range_item(e); VALUE val = rb_range_new(bv, ev, excl); + rb_ractor_make_shareable(rb_obj_freeze(val)); ADD_INSN1(ret, node, putobject, val); RB_OBJ_WRITTEN(iseq, Qundef, val); } @@ -9830,8 +10407,7 @@ compile_kw_arg(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, COMPILE_ERROR(ERROR_ARGS "unreachable"); return COMPILE_NG; } - else if (nd_type_p(default_value, NODE_LIT) || - nd_type_p(default_value, NODE_SYM) || + else if (nd_type_p(default_value, NODE_SYM) || nd_type_p(default_value, NODE_REGX) || nd_type_p(default_value, NODE_LINE) || nd_type_p(default_value, NODE_INTEGER) || @@ -9871,30 +10447,6 @@ compile_attrasgn(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node LABEL *else_label = NULL; VALUE branches = Qfalse; - /* optimization shortcut - * obj["literal"] = value -> opt_aset_with(obj, "literal", value) - */ - if (mid == idASET && !private_recv_p(node) && RNODE_ATTRASGN(node)->nd_args && - nd_type_p(RNODE_ATTRASGN(node)->nd_args, NODE_LIST) && RNODE_LIST(RNODE_ATTRASGN(node)->nd_args)->as.nd_alen == 2 && - (nd_type_p(RNODE_LIST(RNODE_ATTRASGN(node)->nd_args)->nd_head, NODE_STR) || nd_type_p(RNODE_LIST(RNODE_ATTRASGN(node)->nd_args)->nd_head, NODE_FILE)) && - ISEQ_COMPILE_DATA(iseq)->current_block == NULL && - !frozen_string_literal_p(iseq) && - ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) - { - VALUE str = rb_fstring(get_string_value(RNODE_LIST(RNODE_ATTRASGN(node)->nd_args)->nd_head)); - CHECK(COMPILE(ret, "recv", RNODE_ATTRASGN(node)->nd_recv)); - CHECK(COMPILE(ret, "value", RNODE_LIST(RNODE_LIST(RNODE_ATTRASGN(node)->nd_args)->nd_next)->nd_head)); - if (!popped) { - ADD_INSN(ret, node, swap); - ADD_INSN1(ret, node, topn, INT2FIX(1)); - } - ADD_INSN2(ret, node, opt_aset_with, str, - new_callinfo(iseq, idASET, 2, 0, NULL, FALSE)); - RB_OBJ_WRITTEN(iseq, Qundef, str); - ADD_INSN(ret, node, pop); - return COMPILE_OK; - } - INIT_ANCHOR(recv); INIT_ANCHOR(args); argc = setup_args(iseq, args, RNODE_ATTRASGN(node)->nd_args, &flag, NULL); @@ -9917,16 +10469,7 @@ compile_attrasgn(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node ADD_SEQ(ret, recv); ADD_SEQ(ret, args); - if (flag & VM_CALL_ARGS_BLOCKARG) { - ADD_INSN1(ret, node, topn, INT2FIX(1)); - if (flag & VM_CALL_ARGS_SPLAT) { - ADD_INSN1(ret, node, putobject, INT2FIX(-1)); - ADD_SEND_WITH_FLAG(ret, node, idAREF, INT2FIX(1), INT2FIX(asgnflag)); - } - ADD_INSN1(ret, node, setn, FIXNUM_INC(argc, 3)); - ADD_INSN (ret, node, pop); - } - else if (flag & VM_CALL_ARGS_SPLAT) { + if (flag & VM_CALL_ARGS_SPLAT) { ADD_INSN(ret, node, dup); ADD_INSN1(ret, node, putobject, INT2FIX(-1)); ADD_SEND_WITH_FLAG(ret, node, idAREF, INT2FIX(1), INT2FIX(asgnflag)); @@ -9947,6 +10490,375 @@ compile_attrasgn(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node return COMPILE_OK; } +static int +compile_make_shareable_node(rb_iseq_t *iseq, LINK_ANCHOR *ret, LINK_ANCHOR *sub, const NODE *value, bool copy) +{ + ADD_INSN1(ret, value, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + ADD_SEQ(ret, sub); + + if (copy) { + /* + * NEW_CALL(fcore, rb_intern("make_shareable_copy"), + * NEW_LIST(value, loc), loc); + */ + ADD_SEND_WITH_FLAG(ret, value, rb_intern("make_shareable_copy"), INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + else { + /* + * NEW_CALL(fcore, rb_intern("make_shareable"), + * NEW_LIST(value, loc), loc); + */ + ADD_SEND_WITH_FLAG(ret, value, rb_intern("make_shareable"), INT2FIX(1), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + + return COMPILE_OK; +} + +static VALUE +node_const_decl_val(const NODE *node) +{ + VALUE path; + switch (nd_type(node)) { + case NODE_CDECL: + if (RNODE_CDECL(node)->nd_vid) { + path = rb_id2str(RNODE_CDECL(node)->nd_vid); + goto end; + } + else { + node = RNODE_CDECL(node)->nd_else; + } + break; + case NODE_COLON2: + break; + case NODE_COLON3: + // ::Const + path = rb_str_new_cstr("::"); + rb_str_append(path, rb_id2str(RNODE_COLON3(node)->nd_mid)); + goto end; + default: + rb_bug("unexpected node: %s", ruby_node_name(nd_type(node))); + UNREACHABLE_RETURN(0); + } + + path = rb_ary_new(); + if (node) { + for (; node && nd_type_p(node, NODE_COLON2); node = RNODE_COLON2(node)->nd_head) { + rb_ary_push(path, rb_id2str(RNODE_COLON2(node)->nd_mid)); + } + if (node && nd_type_p(node, NODE_CONST)) { + // Const::Name + rb_ary_push(path, rb_id2str(RNODE_CONST(node)->nd_vid)); + } + else if (node && nd_type_p(node, NODE_COLON3)) { + // ::Const::Name + rb_ary_push(path, rb_id2str(RNODE_COLON3(node)->nd_mid)); + rb_ary_push(path, rb_str_new(0, 0)); + } + else { + // expression::Name + rb_ary_push(path, rb_str_new_cstr("...")); + } + path = rb_ary_join(rb_ary_reverse(path), rb_str_new_cstr("::")); + } + end: + path = rb_fstring(path); + return path; +} + +static VALUE +const_decl_path(NODE *dest) +{ + VALUE path = Qnil; + if (!nd_type_p(dest, NODE_CALL)) { + path = node_const_decl_val(dest); + } + return path; +} + +static int +compile_ensure_shareable_node(rb_iseq_t *iseq, LINK_ANCHOR *ret, NODE *dest, const NODE *value) +{ + /* + *. RubyVM::FrozenCore.ensure_shareable(value, const_decl_path(dest)) + */ + VALUE path = const_decl_path(dest); + ADD_INSN1(ret, value, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + CHECK(COMPILE(ret, "compile_ensure_shareable_node", value)); + ADD_INSN1(ret, value, putobject, path); + RB_OBJ_WRITTEN(iseq, Qundef, path); + ADD_SEND_WITH_FLAG(ret, value, rb_intern("ensure_shareable"), INT2FIX(2), INT2FIX(VM_CALL_ARGS_SIMPLE)); + + return COMPILE_OK; +} + +#ifndef SHAREABLE_BARE_EXPRESSION +#define SHAREABLE_BARE_EXPRESSION 1 +#endif + +static int +compile_shareable_literal_constant(rb_iseq_t *iseq, LINK_ANCHOR *ret, enum rb_parser_shareability shareable, NODE *dest, const NODE *node, size_t level, VALUE *value_p, int *shareable_literal_p) +{ +# define compile_shareable_literal_constant_next(node, anchor, value_p, shareable_literal_p) \ + compile_shareable_literal_constant(iseq, anchor, shareable, dest, node, level+1, value_p, shareable_literal_p) + VALUE lit = Qnil; + DECL_ANCHOR(anchor); + + enum node_type type = node ? nd_type(node) : NODE_NIL; + switch (type) { + case NODE_TRUE: + *value_p = Qtrue; + goto compile; + case NODE_FALSE: + *value_p = Qfalse; + goto compile; + case NODE_NIL: + *value_p = Qnil; + goto compile; + case NODE_SYM: + *value_p = rb_node_sym_string_val(node); + goto compile; + case NODE_REGX: + *value_p = rb_node_regx_string_val(node); + goto compile; + case NODE_LINE: + *value_p = rb_node_line_lineno_val(node); + goto compile; + case NODE_INTEGER: + *value_p = rb_node_integer_literal_val(node); + goto compile; + case NODE_FLOAT: + *value_p = rb_node_float_literal_val(node); + goto compile; + case NODE_RATIONAL: + *value_p = rb_node_rational_literal_val(node); + goto compile; + case NODE_IMAGINARY: + *value_p = rb_node_imaginary_literal_val(node); + goto compile; + case NODE_ENCODING: + *value_p = rb_node_encoding_val(node); + + compile: + CHECK(COMPILE(ret, "shareable_literal_constant", node)); + *shareable_literal_p = 1; + return COMPILE_OK; + + case NODE_DSTR: + CHECK(COMPILE(ret, "shareable_literal_constant", node)); + if (shareable == rb_parser_shareable_literal) { + /* + * NEW_CALL(node, idUMinus, 0, loc); + * + * -"#{var}" + */ + ADD_SEND_WITH_FLAG(ret, node, idUMinus, INT2FIX(0), INT2FIX(VM_CALL_ARGS_SIMPLE)); + } + *value_p = Qundef; + *shareable_literal_p = 1; + return COMPILE_OK; + + case NODE_STR:{ + VALUE lit = rb_node_str_string_val(node); + ADD_INSN1(ret, node, putobject, lit); + RB_OBJ_WRITTEN(iseq, Qundef, lit); + *value_p = lit; + *shareable_literal_p = 1; + + return COMPILE_OK; + } + + case NODE_FILE:{ + VALUE lit = rb_node_file_path_val(node); + ADD_INSN1(ret, node, putobject, lit); + RB_OBJ_WRITTEN(iseq, Qundef, lit); + *value_p = lit; + *shareable_literal_p = 1; + + return COMPILE_OK; + } + + case NODE_ZLIST:{ + VALUE lit = rb_ary_new(); + OBJ_FREEZE(lit); + ADD_INSN1(ret, node, putobject, lit); + RB_OBJ_WRITTEN(iseq, Qundef, lit); + *value_p = lit; + *shareable_literal_p = 1; + + return COMPILE_OK; + } + + case NODE_LIST:{ + INIT_ANCHOR(anchor); + lit = rb_ary_new(); + for (NODE *n = (NODE *)node; n; n = RNODE_LIST(n)->nd_next) { + VALUE val; + int shareable_literal_p2; + NODE *elt = RNODE_LIST(n)->nd_head; + if (elt) { + CHECK(compile_shareable_literal_constant_next(elt, anchor, &val, &shareable_literal_p2)); + if (shareable_literal_p2) { + /* noop */ + } + else if (RTEST(lit)) { + rb_ary_clear(lit); + lit = Qfalse; + } + } + if (RTEST(lit)) { + if (!UNDEF_P(val)) { + rb_ary_push(lit, val); + } + else { + rb_ary_clear(lit); + lit = Qnil; /* make shareable at runtime */ + } + } + } + break; + } + case NODE_HASH:{ + if (!RNODE_HASH(node)->nd_brace) { + *value_p = Qundef; + *shareable_literal_p = 0; + return COMPILE_OK; + } + for (NODE *n = RNODE_HASH(node)->nd_head; n; n = RNODE_LIST(RNODE_LIST(n)->nd_next)->nd_next) { + if (!RNODE_LIST(n)->nd_head) { + // If the hash node have a keyword splat, fall back to the default case. + goto compile_shareable; + } + } + + INIT_ANCHOR(anchor); + lit = rb_hash_new(); + for (NODE *n = RNODE_HASH(node)->nd_head; n; n = RNODE_LIST(RNODE_LIST(n)->nd_next)->nd_next) { + VALUE key_val = 0; + VALUE value_val = 0; + int shareable_literal_p2; + NODE *key = RNODE_LIST(n)->nd_head; + NODE *val = RNODE_LIST(RNODE_LIST(n)->nd_next)->nd_head; + CHECK(compile_shareable_literal_constant_next(key, anchor, &key_val, &shareable_literal_p2)); + if (shareable_literal_p2) { + /* noop */ + } + else if (RTEST(lit)) { + rb_hash_clear(lit); + lit = Qfalse; + } + CHECK(compile_shareable_literal_constant_next(val, anchor, &value_val, &shareable_literal_p2)); + if (shareable_literal_p2) { + /* noop */ + } + else if (RTEST(lit)) { + rb_hash_clear(lit); + lit = Qfalse; + } + if (RTEST(lit)) { + if (!UNDEF_P(key_val) && !UNDEF_P(value_val)) { + rb_hash_aset(lit, key_val, value_val); + } + else { + rb_hash_clear(lit); + lit = Qnil; /* make shareable at runtime */ + } + } + } + break; + } + + default: + + compile_shareable: + if (shareable == rb_parser_shareable_literal && + (SHAREABLE_BARE_EXPRESSION || level > 0)) { + CHECK(compile_ensure_shareable_node(iseq, ret, dest, node)); + *value_p = Qundef; + *shareable_literal_p = 1; + return COMPILE_OK; + } + CHECK(COMPILE(ret, "shareable_literal_constant", node)); + *value_p = Qundef; + *shareable_literal_p = 0; + return COMPILE_OK; + } + + /* Array or Hash that does not have keyword splat */ + if (!lit) { + if (nd_type(node) == NODE_LIST) { + ADD_INSN1(anchor, node, newarray, INT2FIX(RNODE_LIST(node)->as.nd_alen)); + } + else if (nd_type(node) == NODE_HASH) { + long len = RNODE_LIST(RNODE_HASH(node)->nd_head)->as.nd_alen; + RBIMPL_ASSERT_OR_ASSUME(len >= 0); + RBIMPL_ASSERT_OR_ASSUME(RB_POSFIXABLE(len)); + ADD_INSN1(anchor, node, newhash, LONG2FIX(len)); + } + *value_p = Qundef; + *shareable_literal_p = 0; + ADD_SEQ(ret, anchor); + return COMPILE_OK; + } + if (NIL_P(lit)) { + // if shareable_literal, all elements should have been ensured + // as shareable + if (nd_type(node) == NODE_LIST) { + ADD_INSN1(anchor, node, newarray, INT2FIX(RNODE_LIST(node)->as.nd_alen)); + } + else if (nd_type(node) == NODE_HASH) { + long len = RNODE_LIST(RNODE_HASH(node)->nd_head)->as.nd_alen; + RBIMPL_ASSERT_OR_ASSUME(len >= 0); + RBIMPL_ASSERT_OR_ASSUME(RB_POSFIXABLE(len)); + ADD_INSN1(anchor, node, newhash, LONG2FIX(len)); + } + CHECK(compile_make_shareable_node(iseq, ret, anchor, node, false)); + *value_p = Qundef; + *shareable_literal_p = 1; + } + else { + VALUE val = rb_ractor_make_shareable(lit); + ADD_INSN1(ret, node, putobject, val); + RB_OBJ_WRITTEN(iseq, Qundef, val); + *value_p = val; + *shareable_literal_p = 1; + } + + return COMPILE_OK; +} + +static int +compile_shareable_constant_value(rb_iseq_t *iseq, LINK_ANCHOR *ret, enum rb_parser_shareability shareable, const NODE *lhs, const NODE *value) +{ + int literal_p = 0; + VALUE val; + DECL_ANCHOR(anchor); + INIT_ANCHOR(anchor); + + switch (shareable) { + case rb_parser_shareable_none: + CHECK(COMPILE(ret, "compile_shareable_constant_value", value)); + return COMPILE_OK; + + case rb_parser_shareable_literal: + CHECK(compile_shareable_literal_constant(iseq, anchor, shareable, (NODE *)lhs, value, 0, &val, &literal_p)); + ADD_SEQ(ret, anchor); + return COMPILE_OK; + + case rb_parser_shareable_copy: + case rb_parser_shareable_everything: + CHECK(compile_shareable_literal_constant(iseq, anchor, shareable, (NODE *)lhs, value, 0, &val, &literal_p)); + if (!literal_p) { + CHECK(compile_make_shareable_node(iseq, ret, anchor, value, shareable == rb_parser_shareable_copy)); + } + else { + ADD_SEQ(ret, anchor); + } + return COMPILE_OK; + default: + rb_bug("unexpected rb_parser_shareability: %d", shareable); + } +} + static int iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, int popped); /** compile each node @@ -9984,7 +10896,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no if (nd_fl_newline(node)) { int event = RUBY_EVENT_LINE; ISEQ_COMPILE_DATA(iseq)->last_line = line; - if (ISEQ_COVERAGE(iseq) && ISEQ_LINE_COVERAGE(iseq)) { + if (line > 0 && ISEQ_COVERAGE(iseq) && ISEQ_LINE_COVERAGE(iseq)) { event |= RUBY_EVENT_COVERAGE_LINE; } ADD_TRACE(ret, event); @@ -10129,7 +11041,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no } case NODE_CDECL:{ if (RNODE_CDECL(node)->nd_vid) { - CHECK(COMPILE(ret, "lvalue", RNODE_CDECL(node)->nd_value)); + CHECK(compile_shareable_constant_value(iseq, ret, RNODE_CDECL(node)->shareability, node, RNODE_CDECL(node)->nd_value)); if (!popped) { ADD_INSN(ret, node, dup); @@ -10141,7 +11053,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no } else { compile_cpath(ret, iseq, RNODE_CDECL(node)->nd_else); - CHECK(COMPILE(ret, "lvalue", RNODE_CDECL(node)->nd_value)); + CHECK(compile_shareable_constant_value(iseq, ret, RNODE_CDECL(node)->shareability, node, RNODE_CDECL(node)->nd_value)); ADD_INSN(ret, node, swap); if (!popped) { @@ -10253,6 +11165,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { body->ic_size++; VALUE segments = rb_ary_new_from_args(1, ID2SYM(RNODE_CONST(node)->nd_vid)); + RB_OBJ_SET_FROZEN_SHAREABLE(segments); ADD_INSN1(ret, node, opt_getconstant_path, segments); RB_OBJ_WRITTEN(iseq, Qundef, segments); } @@ -10298,14 +11211,6 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no case NODE_MATCH3: CHECK(compile_match(iseq, ret, node, popped, type)); break; - case NODE_LIT:{ - debugp_param("lit", RNODE_LIT(node)->nd_lit); - if (!popped) { - ADD_INSN1(ret, node, putobject, RNODE_LIT(node)->nd_lit); - RB_OBJ_WRITTEN(iseq, Qundef, RNODE_LIT(node)->nd_lit); - } - break; - } case NODE_SYM:{ if (!popped) { ADD_INSN1(ret, node, putobject, rb_node_sym_string_val(node)); @@ -10326,6 +11231,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no } case NODE_INTEGER:{ VALUE lit = rb_node_integer_literal_val(node); + if (!SPECIAL_CONST_P(lit)) RB_OBJ_SET_SHAREABLE(lit); debugp_param("integer", lit); if (!popped) { ADD_INSN1(ret, node, putobject, lit); @@ -10335,6 +11241,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no } case NODE_FLOAT:{ VALUE lit = rb_node_float_literal_val(node); + if (!SPECIAL_CONST_P(lit)) RB_OBJ_SET_SHAREABLE(lit); debugp_param("float", lit); if (!popped) { ADD_INSN1(ret, node, putobject, lit); @@ -10344,6 +11251,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no } case NODE_RATIONAL:{ VALUE lit = rb_node_rational_literal_val(node); + rb_ractor_make_shareable(lit); debugp_param("rational", lit); if (!popped) { ADD_INSN1(ret, node, putobject, lit); @@ -10353,6 +11261,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no } case NODE_IMAGINARY:{ VALUE lit = rb_node_imaginary_literal_val(node); + rb_ractor_make_shareable(lit); debugp_param("imaginary", lit); if (!popped) { ADD_INSN1(ret, node, putobject, lit); @@ -10365,33 +11274,26 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no debugp_param("nd_lit", get_string_value(node)); if (!popped) { VALUE lit = get_string_value(node); - switch (ISEQ_COMPILE_DATA(iseq)->option->frozen_string_literal) { + const rb_compile_option_t *option = ISEQ_COMPILE_DATA(iseq)->option; + if ((option->debug_frozen_string_literal || RTEST(ruby_debug)) && + option->frozen_string_literal != ISEQ_FROZEN_STRING_LITERAL_DISABLED) { + lit = rb_str_with_debug_created_info(lit, rb_iseq_path(iseq), line); + RB_OBJ_SET_SHAREABLE(lit); + } + switch (option->frozen_string_literal) { case ISEQ_FROZEN_STRING_LITERAL_UNSET: - lit = rb_fstring(lit); - ADD_INSN1(ret, node, putchilledstring, lit); - RB_OBJ_WRITTEN(iseq, Qundef, lit); + ADD_INSN1(ret, node, dupchilledstring, lit); break; case ISEQ_FROZEN_STRING_LITERAL_DISABLED: - lit = rb_fstring(lit); - ADD_INSN1(ret, node, putstring, lit); - RB_OBJ_WRITTEN(iseq, Qundef, lit); + ADD_INSN1(ret, node, dupstring, lit); break; case ISEQ_FROZEN_STRING_LITERAL_ENABLED: - if (ISEQ_COMPILE_DATA(iseq)->option->debug_frozen_string_literal || RTEST(ruby_debug)) { - VALUE debug_info = rb_ary_new_from_args(2, rb_iseq_path(iseq), INT2FIX(line)); - lit = rb_str_dup(lit); - rb_ivar_set(lit, id_debug_created_info, rb_obj_freeze(debug_info)); - lit = rb_str_freeze(lit); - } - else { - lit = rb_fstring(lit); - } ADD_INSN1(ret, node, putobject, lit); - RB_OBJ_WRITTEN(iseq, Qundef, lit); break; default: rb_bug("invalid frozen_string_literal"); } + RB_OBJ_WRITTEN(iseq, Qundef, lit); } break; } @@ -10405,7 +11307,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no } case NODE_XSTR:{ ADD_CALL_RECEIVER(ret, node); - VALUE str = rb_fstring(rb_node_str_string_val(node)); + VALUE str = rb_node_str_string_val(node); ADD_INSN1(ret, node, putobject, str); RB_OBJ_WRITTEN(iseq, Qundef, str); ADD_CALL(ret, node, idBackquote, INT2FIX(1)); @@ -10431,6 +11333,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no case NODE_REGX:{ if (!popped) { VALUE lit = rb_node_regx_string_val(node); + RB_OBJ_SET_SHAREABLE(lit); ADD_INSN1(ret, node, putobject, lit); RB_OBJ_WRITTEN(iseq, Qundef, lit); } @@ -10564,10 +11467,18 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no break; } case NODE_UNDEF:{ - ADD_INSN1(ret, node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); - ADD_INSN1(ret, node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CBASE)); - CHECK(COMPILE(ret, "undef arg", RNODE_UNDEF(node)->nd_undef)); - ADD_SEND(ret, node, id_core_undef_method, INT2FIX(2)); + const rb_parser_ary_t *ary = RNODE_UNDEF(node)->nd_undefs; + + for (long i = 0; i < ary->len; i++) { + ADD_INSN1(ret, node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + ADD_INSN1(ret, node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CBASE)); + CHECK(COMPILE(ret, "undef arg", ary->data[i])); + ADD_SEND(ret, node, id_core_undef_method, INT2FIX(2)); + + if (i < ary->len - 1) { + ADD_INSN(ret, node, pop); + } + } if (popped) { ADD_INSN(ret, node, pop); @@ -10615,9 +11526,20 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no CHECK(COMPILE(ret, "sclass#recv", RNODE_SCLASS(node)->nd_recv)); ADD_INSN (ret, node, putnil); CONST_ID(singletonclass, "singletonclass"); + + /* `class << self` in a class body and `class << Foo` (constant + receiver) are stable. All other forms are potentially dynamic. */ + int sclass_flags = VM_DEFINECLASS_TYPE_SINGLETON_CLASS; + const NODE *recv = RNODE_SCLASS(node)->nd_recv; + if (!(nd_type_p(recv, NODE_SELF) && + ISEQ_BODY(iseq)->type == ISEQ_TYPE_CLASS) && + !cpath_const_p(recv)) { + sclass_flags |= VM_DEFINECLASS_FLAG_DYNAMIC_CREF; + } + ADD_INSN3(ret, node, defineclass, ID2SYM(singletonclass), singleton_class, - INT2FIX(VM_DEFINECLASS_TYPE_SINGLETON_CLASS)); + INT2FIX(sclass_flags)); RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)singleton_class); if (popped) { @@ -10681,7 +11603,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no break; case NODE_DEFINED: if (!popped) { - CHECK(compile_defined_expr(iseq, ret, node, Qtrue)); + CHECK(compile_defined_expr(iseq, ret, node, Qtrue, false)); } break; case NODE_POSTEXE:{ @@ -10692,8 +11614,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no struct rb_iseq_new_with_callback_callback_func *ifunc = rb_iseq_new_with_callback_new_callback(build_postexe_iseq, RNODE_POSTEXE(node)->nd_body); const rb_iseq_t *once_iseq = - new_child_iseq_with_callback(iseq, ifunc, - rb_fstring(make_name_for_block(iseq)), iseq, ISEQ_TYPE_BLOCK, line); + NEW_CHILD_ISEQ_WITH_CALLBACK(ifunc, rb_fstring(make_name_for_block(iseq)), ISEQ_TYPE_BLOCK, line); ADD_INSN2(ret, node, once, once_iseq, INT2FIX(is_index)); RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)once_iseq); @@ -10841,7 +11762,7 @@ insn_data_to_s_detail(INSN *iobj) { const struct rb_callinfo *ci = (struct rb_callinfo *)OPERAND_AT(iobj, j); rb_str_cat2(str, "<calldata:"); - if (vm_ci_mid(ci)) rb_str_catf(str, "%"PRIsVALUE, rb_id2str(vm_ci_mid(ci))); + if (vm_ci_mid(ci)) rb_str_append(str, rb_id2str(vm_ci_mid(ci))); rb_str_catf(str, ", %d>", vm_ci_argc(ci)); break; } @@ -10924,7 +11845,7 @@ dump_disasm_list_with_cursor(const LINK_ELEMENT *link, const LINK_ELEMENT *curr, } default: /* ignore */ - rb_raise(rb_eSyntaxError, "dump_disasm_list error: %ld\n", FIX2LONG(link->type)); + rb_raise(rb_eSyntaxError, "dump_disasm_list error: %d\n", (int)link->type); } link = link->next; } @@ -10952,7 +11873,7 @@ rb_insns_name_array(void) for (i = 0; i < VM_INSTRUCTION_SIZE; i++) { rb_ary_push(ary, rb_fstring_cstr(insn_name(i))); } - return rb_obj_freeze(ary); + return rb_ary_freeze(ary); } static LABEL * @@ -11138,7 +12059,7 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, { /* TODO: body should be frozen */ long i, len = RARRAY_LEN(body); - struct st_table *labels_table = DATA_PTR(labels_wrapper); + struct st_table *labels_table = RTYPEDDATA_DATA(labels_wrapper); int j; int line_no = 0, node_id = -1, insn_idx = 0; int ret = COMPILE_OK; @@ -11195,7 +12116,7 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, } if (argc > 0) { - argv = compile_data_calloc2(iseq, sizeof(VALUE), argc); + argv = compile_data_calloc2_type(iseq, VALUE, argc); // add element before operand setup to make GC root ADD_ELEM(anchor, @@ -11275,18 +12196,18 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, case TS_CDHASH: { int i; - VALUE map = rb_hash_new_with_size(RARRAY_LEN(op)/2); + VALUE map = cdhash_new(RARRAY_LEN(op) / 2); - RHASH_TBL_RAW(map)->type = &cdhash_type; op = rb_to_array_type(op); for (i=0; i<RARRAY_LEN(op); i+=2) { VALUE key = RARRAY_AREF(op, i); VALUE sym = RARRAY_AREF(op, i+1); LABEL *label = register_label(iseq, labels_table, sym); - rb_hash_aset(map, key, (VALUE)label | 1); + cdhash_aset(map, key, (VALUE)label); } RB_GC_GUARD(op); + RB_OBJ_SET_SHAREABLE(map); // allow mutation while compiling argv[j] = map; RB_OBJ_WRITTEN(iseq, Qundef, map); } @@ -11316,7 +12237,8 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, rb_raise(rb_eTypeError, "unexpected object for instruction"); } } - DATA_PTR(labels_wrapper) = 0; + RTYPEDDATA_DATA(labels_wrapper) = 0; + RB_GC_GUARD(labels_wrapper); validate_labels(iseq, labels_table); if (!ret) return ret; return iseq_setup(iseq, anchor); @@ -11400,7 +12322,7 @@ iseq_build_kw(rb_iseq_t *iseq, VALUE params, VALUE keywords) rb_raise(rb_eTypeError, "keyword default has unsupported len %+"PRIsVALUE, key); } ids[i] = SYM2ID(sym); - dvs[j] = default_val; + RB_OBJ_WRITE(iseq, &dvs[j], default_val); } keyword->table = ids; @@ -11410,44 +12332,48 @@ iseq_build_kw(rb_iseq_t *iseq, VALUE params, VALUE keywords) } static void -iseq_insn_each_object_mark_and_pin(VALUE obj, VALUE _) +iseq_insn_each_object_mark_and_move(VALUE * obj, VALUE _) { - rb_gc_mark(obj); + rb_gc_mark_and_move(obj); } void -rb_iseq_mark_and_pin_insn_storage(struct iseq_compile_data_storage *storage) +rb_iseq_mark_and_move_insn_storage(struct iseq_compile_data_storage *storage) { INSN *iobj = 0; size_t size = sizeof(INSN); + size_t align = ALIGNMENT_SIZE_OF(INSN); unsigned int pos = 0; while (storage) { -#ifdef STRICT_ALIGNMENT - size_t padding = calc_padding((void *)&storage->buff[pos], size); -#else - const size_t padding = 0; /* expected to be optimized by compiler */ -#endif /* STRICT_ALIGNMENT */ + size_t padding = calc_padding((void *)&storage->buff[pos], align); size_t offset = pos + size + padding; if (offset > storage->size || offset > storage->pos) { pos = 0; storage = storage->next; } else { -#ifdef STRICT_ALIGNMENT pos += (int)padding; -#endif /* STRICT_ALIGNMENT */ iobj = (INSN *)&storage->buff[pos]; if (iobj->operands) { - iseq_insn_each_markable_object(iobj, iseq_insn_each_object_mark_and_pin, (VALUE)0); + iseq_insn_each_markable_object(iobj, iseq_insn_each_object_mark_and_move, (VALUE)0); } pos += (int)size; } } } +static const rb_data_type_t labels_wrapper_type = { + .wrap_struct_name = "compiler/labels_wrapper", + .function = { + .dmark = (RUBY_DATA_FUNC)rb_mark_set, + .dfree = (RUBY_DATA_FUNC)st_free_table, + }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED, +}; + void rb_iseq_build_from_ary(rb_iseq_t *iseq, VALUE misc, VALUE locals, VALUE params, VALUE exception, VALUE body) @@ -11457,7 +12383,7 @@ rb_iseq_build_from_ary(rb_iseq_t *iseq, VALUE misc, VALUE locals, VALUE params, unsigned int arg_size, local_size, stack_max; ID *tbl; struct st_table *labels_table = st_init_numtable(); - VALUE labels_wrapper = Data_Wrap_Struct(0, rb_mark_set, st_free_table, labels_table); + VALUE labels_wrapper = TypedData_Wrap_Struct(0, &labels_wrapper_type, labels_table); VALUE arg_opt_labels = rb_hash_aref(params, SYM(opt)); VALUE keywords = rb_hash_aref(params, SYM(keyword)); VALUE sym_arg_rest = ID2SYM(rb_intern_const("#arg_rest")); @@ -11539,6 +12465,10 @@ rb_iseq_build_from_ary(rb_iseq_t *iseq, VALUE misc, VALUE locals, VALUE params, ISEQ_BODY(iseq)->param.flags.ambiguous_param0 = TRUE; } + if (Qtrue == rb_hash_aref(params, SYM(use_block))) { + ISEQ_BODY(iseq)->param.flags.use_block = TRUE; + } + if (int_param(&i, params, SYM(kwrest))) { struct rb_iseq_param_keyword *keyword = (struct rb_iseq_param_keyword *)ISEQ_BODY(iseq)->param.keyword; if (keyword == NULL) { @@ -11619,7 +12549,7 @@ typedef uint32_t ibf_offset_t; #define IBF_MAJOR_VERSION ISEQ_MAJOR_VERSION #ifdef RUBY_DEVEL -#define IBF_DEVEL_VERSION 4 +#define IBF_DEVEL_VERSION 5 #define IBF_MINOR_VERSION (ISEQ_MINOR_VERSION * 10000 + IBF_DEVEL_VERSION) #else #define IBF_MINOR_VERSION ISEQ_MINOR_VERSION @@ -11778,8 +12708,13 @@ static ibf_offset_t ibf_dump_write(struct ibf_dump *dump, const void *buff, unsigned long size) { ibf_offset_t pos = ibf_dump_pos(dump); +#if SIZEOF_LONG > SIZEOF_INT + /* ensure the resulting dump does not exceed UINT_MAX */ + if (size >= UINT_MAX || pos + size >= UINT_MAX) { + rb_raise(rb_eRuntimeError, "dump size exceeds"); + } +#endif rb_str_cat(dump->current_buffer->str, (const char *)buff, size); - /* TODO: overflow check */ return pos; } @@ -12017,7 +12952,8 @@ ibf_load_builtin(const struct ibf_load *load, ibf_offset_t *offset) const struct rb_builtin_function *table = GET_VM()->builtin_function_table; if (table == NULL) rb_raise(rb_eArgError, "builtin function table is not provided"); if (strncmp(table[i].name, name, len) != 0) { - rb_raise(rb_eArgError, "builtin function index (%d) mismatch (expect %s but %s)", i, name, table[i].name); + rb_raise(rb_eArgError, "builtin function index (%d) mismatch (expect %.*s but %s)", + i, len, name, table[i].name); } // fprintf(stderr, "load-builtin: name:%s(%d)\n", table[i].name, table[i].argc); @@ -12103,21 +13039,23 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod VALUE iseqv = (VALUE)iseq; unsigned int code_index; ibf_offset_t reading_pos = bytecode_offset; - VALUE *code = ALLOC_N(VALUE, iseq_size); + VALUE *code = ZALLOC_N(VALUE, iseq_size); struct rb_iseq_constant_body *load_body = ISEQ_BODY(iseq); struct rb_call_data *cd_entries = load_body->call_data; int ic_index = 0; - iseq_bits_t * mark_offset_bits; - - iseq_bits_t tmp[1] = {0}; + load_body->iseq_encoded = code; + load_body->iseq_size = iseq_size; + iseq_bits_t * mark_offset_bits; if (ISEQ_MBITS_BUFLEN(iseq_size) == 1) { - mark_offset_bits = tmp; + load_body->mark_bits.single = 0; + mark_offset_bits = &load_body->mark_bits.single; } else { - mark_offset_bits = ZALLOC_N(iseq_bits_t, ISEQ_MBITS_BUFLEN(iseq_size)); + load_body->mark_bits.list = ZALLOC_N(iseq_bits_t, ISEQ_MBITS_BUFLEN(iseq_size)); + mark_offset_bits = load_body->mark_bits.list; } bool needs_bitmap = false; @@ -12149,10 +13087,6 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod { VALUE op = ibf_load_small_value(load, &reading_pos); VALUE v = ibf_load_object(load, op); - v = rb_hash_dup(v); // hash dumped as frozen - RHASH_TBL_RAW(v)->type = &cdhash_type; - rb_hash_rehash(v); // hash function changed - freeze_hide_obj(v); // Overwrite the existing hash in the object list. This // is to keep the object alive during load time. @@ -12203,12 +13137,12 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod if (insn == BIN(setinstancevariable)) { ID iv_name = (ID)code[code_index - 1]; cache->iv_set_name = iv_name; + cache->value = IVAR_CACHE_INIT; } else { cache->iv_set_name = 0; + cache->value = rb_getivar_cache_pack(ROOT_SHAPE_ID, ATTR_INDEX_NOT_SET); } - - vm_ic_attr_index_initialize(cache, INVALID_SHAPE_ID); } } @@ -12240,20 +13174,9 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod } } - load_body->iseq_encoded = code; - load_body->iseq_size = code_index; - - if (ISEQ_MBITS_BUFLEN(load_body->iseq_size) == 1) { - load_body->mark_bits.single = mark_offset_bits[0]; - } - else { - if (needs_bitmap) { - load_body->mark_bits.list = mark_offset_bits; - } - else { - load_body->mark_bits.list = 0; - ruby_xfree(mark_offset_bits); - } + if (!needs_bitmap) { + SIZED_FREE_N(load_body->mark_bits.list, ISEQ_MBITS_BUFLEN(iseq_size)); + load_body->mark_bits.list = NULL; } RUBY_ASSERT(code_index == iseq_size); @@ -12318,19 +13241,17 @@ ibf_load_param_keyword(const struct ibf_load *load, ibf_offset_t param_keyword_o { if (param_keyword_offset) { struct rb_iseq_param_keyword *kw = IBF_R(param_keyword_offset, struct rb_iseq_param_keyword, 1); - ID *ids = IBF_R(kw->table, ID, kw->num); int dv_num = kw->num - kw->required_num; - VALUE *dvs = IBF_R(kw->default_values, VALUE, dv_num); - int i; + VALUE *dvs = dv_num ? IBF_R(kw->default_values, VALUE, dv_num) : NULL; - for (i=0; i<kw->num; i++) { - ids[i] = ibf_load_id(load, ids[i]); - } + int i; for (i=0; i<dv_num; i++) { dvs[i] = ibf_load_object(load, dvs[i]); } - kw->table = ids; + // Will be set once the local table is loaded. + kw->table = NULL; + kw->default_values = dvs; return kw; } @@ -12427,7 +13348,7 @@ ibf_dump_local_table(struct ibf_dump *dump, const rb_iseq_t *iseq) return ibf_dump_write(dump, table, sizeof(ID) * size); } -static ID * +static const ID * ibf_load_local_table(const struct ibf_load *load, ibf_offset_t local_table_offset, int size) { if (size > 0) { @@ -12437,7 +13358,14 @@ ibf_load_local_table(const struct ibf_load *load, ibf_offset_t local_table_offse for (i=0; i<size; i++) { table[i] = ibf_load_id(load, table[i]); } - return table; + + if (size == 1 && table[0] == idERROR_INFO) { + ruby_xfree_sized(table, sizeof(ID) * size); + return rb_iseq_shared_exc_local_tbl; + } + else { + return table; + } } else { return NULL; @@ -12445,6 +13373,28 @@ ibf_load_local_table(const struct ibf_load *load, ibf_offset_t local_table_offse } static ibf_offset_t +ibf_dump_lvar_states(struct ibf_dump *dump, const rb_iseq_t *iseq) +{ + const struct rb_iseq_constant_body *const body = ISEQ_BODY(iseq); + const int size = body->local_table_size; + IBF_W_ALIGN(enum lvar_state); + return ibf_dump_write(dump, body->lvar_states, sizeof(enum lvar_state) * (body->lvar_states ? size : 0)); +} + +static enum lvar_state * +ibf_load_lvar_states(const struct ibf_load *load, ibf_offset_t lvar_states_offset, int size, const ID *local_table) +{ + if (local_table == rb_iseq_shared_exc_local_tbl || + size <= 0) { + return NULL; + } + else { + enum lvar_state *states = IBF_R(lvar_states_offset, enum lvar_state, size); + return states; + } +} + +static ibf_offset_t ibf_dump_catch_table(struct ibf_dump *dump, const rb_iseq_t *iseq) { const struct iseq_catch_table *table = ISEQ_BODY(iseq)->catch_table; @@ -12474,12 +13424,13 @@ ibf_dump_catch_table(struct ibf_dump *dump, const rb_iseq_t *iseq) } } -static struct iseq_catch_table * -ibf_load_catch_table(const struct ibf_load *load, ibf_offset_t catch_table_offset, unsigned int size) +static void +ibf_load_catch_table(const struct ibf_load *load, ibf_offset_t catch_table_offset, unsigned int size, const rb_iseq_t *parent_iseq) { if (size) { - struct iseq_catch_table *table = ruby_xmalloc(iseq_catch_table_bytes(size)); + struct iseq_catch_table *table = ruby_xcalloc(1, iseq_catch_table_bytes(size)); table->size = size; + ISEQ_BODY(parent_iseq)->catch_table = table; ibf_offset_t reading_pos = catch_table_offset; @@ -12492,12 +13443,12 @@ ibf_load_catch_table(const struct ibf_load *load, ibf_offset_t catch_table_offse table->entries[i].cont = (unsigned int)ibf_load_small_value(load, &reading_pos); table->entries[i].sp = (unsigned int)ibf_load_small_value(load, &reading_pos); - table->entries[i].iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)iseq_index); + rb_iseq_t *catch_iseq = (rb_iseq_t *)ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)iseq_index); + RB_OBJ_WRITE(parent_iseq, UNALIGNED_MEMBER_PTR(&table->entries[i], iseq), catch_iseq); } - return table; } else { - return NULL; + ISEQ_BODY(parent_iseq)->catch_table = NULL; } } @@ -12568,6 +13519,14 @@ outer_variable_cmp(const void *a, const void *b, void *arg) { const struct outer_variable_pair *ap = (const struct outer_variable_pair *)a; const struct outer_variable_pair *bp = (const struct outer_variable_pair *)b; + + if (!ap->name) { + return -1; + } + else if (!bp->name) { + return 1; + } + return rb_str_cmp(ap->name, bp->name); } @@ -12608,6 +13567,11 @@ ibf_load_ci_entries(const struct ibf_load *load, unsigned int ci_size, struct rb_call_data **cd_ptr) { + if (!ci_size) { + *cd_ptr = NULL; + return; + } + ibf_offset_t reading_pos = ci_entries_offset; unsigned int i; @@ -12700,9 +13664,10 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq) positions = rb_iseq_insns_info_decode_positions(ISEQ_BODY(iseq)); const ibf_offset_t insns_info_positions_offset = ibf_dump_insns_info_positions(dump, positions, body->insns_info.size); - ruby_xfree(positions); + SIZED_FREE_N(positions, ISEQ_BODY(iseq)->insns_info.size); const ibf_offset_t local_table_offset = ibf_dump_local_table(dump, iseq); + const ibf_offset_t lvar_states_offset = ibf_dump_lvar_states(dump, iseq); const unsigned int catch_table_size = body->catch_table ? body->catch_table->size : 0; const ibf_offset_t catch_table_offset = ibf_dump_catch_table(dump, iseq); const int parent_iseq_index = ibf_dump_iseq(dump, ISEQ_BODY(iseq)->parent_iseq); @@ -12731,7 +13696,12 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq) (body->param.flags.has_block << 6) | (body->param.flags.ambiguous_param0 << 7) | (body->param.flags.accepts_no_kwarg << 8) | - (body->param.flags.ruby2_keywords << 9); + (body->param.flags.ruby2_keywords << 9) | + (body->param.flags.anon_rest << 10) | + (body->param.flags.anon_kwrest << 11) | + (body->param.flags.use_block << 12) | + (body->param.flags.forwardable << 13) | + (body->param.flags.accepts_no_block << 14); #if IBF_ISEQ_ENABLE_LOCAL_BUFFER # define IBF_BODY_OFFSET(x) (x) @@ -12766,6 +13736,7 @@ ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq) ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(insns_info_positions_offset)); ibf_dump_write_small_value(dump, body->insns_info.size); ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(local_table_offset)); + ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(lvar_states_offset)); ibf_dump_write_small_value(dump, catch_table_size); ibf_dump_write_small_value(dump, IBF_BODY_OFFSET(catch_table_offset)); ibf_dump_write_small_value(dump, parent_iseq_index); @@ -12877,6 +13848,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset) const ibf_offset_t insns_info_positions_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos)); const unsigned int insns_info_size = (unsigned int)ibf_load_small_value(load, &reading_pos); const ibf_offset_t local_table_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos)); + const ibf_offset_t lvar_states_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos)); const unsigned int catch_table_size = (unsigned int)ibf_load_small_value(load, &reading_pos); const ibf_offset_t catch_table_offset = (ibf_offset_t)IBF_BODY_OFFSET(ibf_load_small_value(load, &reading_pos)); const int parent_iseq_index = (int)ibf_load_small_value(load, &reading_pos); @@ -12947,6 +13919,9 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset) load_body->param.flags.ruby2_keywords = (param_flags >> 9) & 1; load_body->param.flags.anon_rest = (param_flags >> 10) & 1; load_body->param.flags.anon_kwrest = (param_flags >> 11) & 1; + load_body->param.flags.use_block = (param_flags >> 12) & 1; + load_body->param.flags.forwardable = (param_flags >> 13) & 1; + load_body->param.flags.accepts_no_block = (param_flags >> 14) & 1; load_body->param.size = param_size; load_body->param.lead_num = param_lead_num; load_body->param.opt_num = param_opt_num; @@ -12991,10 +13966,23 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset) load_body->insns_info.body = ibf_load_insns_info_body(load, insns_info_body_offset, insns_info_size); load_body->insns_info.positions = ibf_load_insns_info_positions(load, insns_info_positions_offset, insns_info_size); load_body->local_table = ibf_load_local_table(load, local_table_offset, local_table_size); - load_body->catch_table = ibf_load_catch_table(load, catch_table_offset, catch_table_size); - load_body->parent_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)parent_iseq_index); - load_body->local_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)local_iseq_index); - load_body->mandatory_only_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)mandatory_only_iseq_index); + load_body->lvar_states = ibf_load_lvar_states(load, lvar_states_offset, local_table_size, load_body->local_table); + ibf_load_catch_table(load, catch_table_offset, catch_table_size, iseq); + + const rb_iseq_t *parent_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)parent_iseq_index); + const rb_iseq_t *local_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)local_iseq_index); + const rb_iseq_t *mandatory_only_iseq = ibf_load_iseq(load, (const rb_iseq_t *)(VALUE)mandatory_only_iseq_index); + + RB_OBJ_WRITE(iseq, &load_body->parent_iseq, parent_iseq); + RB_OBJ_WRITE(iseq, &load_body->local_iseq, local_iseq); + RB_OBJ_WRITE(iseq, &load_body->mandatory_only_iseq, mandatory_only_iseq); + + // This must be done after the local table is loaded. + if (load_body->param.keyword != NULL) { + RUBY_ASSERT(load_body->local_table); + struct rb_iseq_param_keyword *keyword = (struct rb_iseq_param_keyword *) load_body->param.keyword; + keyword->table = &load_body->local_table[keyword->bits_start - keyword->num]; + } ibf_load_code(load, iseq, bytecode_offset, bytecode_size, iseq_size); #if VM_INSN_INFO_TABLE_IMPL == 2 @@ -13061,8 +14049,6 @@ ibf_dump_iseq_list(struct ibf_dump *dump, struct ibf_header *header) header->iseq_list_size = (unsigned int)size; } -#define IBF_OBJECT_INTERNAL FL_PROMOTED0 - /* * Binary format * - ibf_object_header @@ -13122,7 +14108,11 @@ struct ibf_object_symbol { #define IBF_ALIGNED_OFFSET(align, offset) /* offset > 0 */ \ ((((offset) - 1) / (align) + 1) * (align)) -#define IBF_OBJBODY(type, offset) (const type *)\ +/* No cast, since it's UB to create an unaligned pointer. + * Leave as void* for use with memcpy in those cases. + * We align the offset, but the buffer pointer is only VALUE aligned, + * so the returned pointer may be unaligned for `type` .*/ +#define IBF_OBJBODY(type, offset) \ ibf_load_check_offset(load, IBF_ALIGNED_OFFSET(RUBY_ALIGNOF(type), offset)) static const void * @@ -13217,8 +14207,12 @@ ibf_dump_object_float(struct ibf_dump *dump, VALUE obj) static VALUE ibf_load_object_float(const struct ibf_load *load, const struct ibf_object_header *header, ibf_offset_t offset) { - const double *dblp = IBF_OBJBODY(double, offset); - return DBL2NUM(*dblp); + double d; + /* Avoid unaligned VFP load on ARMv7; IBF payload may be unaligned (C99 6.3.2.3 p7). */ + memcpy(&d, IBF_OBJBODY(double, offset), sizeof(d)); + VALUE f = DBL2NUM(d); + if (!FLONUM_P(f)) RB_OBJ_SET_SHAREABLE(f); + return f; } static void @@ -13255,7 +14249,7 @@ ibf_load_object_string(const struct ibf_load *load, const struct ibf_object_head VALUE str; if (header->frozen && !header->internal) { - str = rb_enc_interned_str(ptr, len, rb_enc_from_index(encindex)); + str = rb_enc_literal_str(ptr, len, rb_enc_from_index(encindex)); } else { str = rb_enc_str_new(ptr, len, rb_enc_from_index(encindex)); @@ -13289,7 +14283,7 @@ ibf_load_object_regexp(const struct ibf_load *load, const struct ibf_object_head VALUE reg = rb_reg_compile(srcstr, (int)regexp.option, NULL, 0); if (header->internal) rb_obj_hide(reg); - if (header->frozen) rb_obj_freeze(reg); + if (header->frozen) RB_OBJ_SET_SHAREABLE(rb_obj_freeze(reg)); return reg; } @@ -13320,7 +14314,10 @@ ibf_load_object_array(const struct ibf_load *load, const struct ibf_object_heade rb_ary_push(ary, ibf_load_object(load, index)); } - if (header->frozen) rb_obj_freeze(ary); + if (header->frozen) { + rb_ary_freeze(ary); + rb_ractor_make_shareable(ary); // TODO: check elements + } return ary; } @@ -13351,7 +14348,7 @@ static VALUE ibf_load_object_hash(const struct ibf_load *load, const struct ibf_object_header *header, ibf_offset_t offset) { long len = (long)ibf_load_small_value(load, &offset); - VALUE obj = rb_hash_new_with_size(len); + VALUE obj = header->frozen ? rb_hash_alloc_fixed_size(rb_cHash, len) : rb_hash_new_with_size(len); int i; for (i = 0; i < len; i++) { @@ -13362,10 +14359,59 @@ ibf_load_object_hash(const struct ibf_load *load, const struct ibf_object_header VALUE val = ibf_load_object(load, val_index); rb_hash_aset(obj, key, val); } - rb_hash_rehash(obj); if (header->internal) rb_obj_hide(obj); - if (header->frozen) rb_obj_freeze(obj); + if (header->frozen) { + RB_OBJ_SET_FROZEN_SHAREABLE(obj); + } + + return obj; +} + +static int +ibf_dump_cdhash_i(st_data_t key, st_data_t val, st_data_t ptr) +{ + struct ibf_dump *dump = (struct ibf_dump *)ptr; + + VALUE key_index = ibf_dump_object(dump, (VALUE)key); + + ibf_dump_write_small_value(dump, key_index); + ibf_dump_write_small_value(dump, val); + return ST_CONTINUE; +} + +static void +ibf_dump_object_imemo(struct ibf_dump *dump, VALUE obj) +{ + switch (imemo_type(obj)) { + case imemo_cdhash: { + st_table *tbl = rb_imemo_cdhash_tbl(obj); + + long len = tbl->num_entries; + ibf_dump_write_small_value(dump, (VALUE)len); + if (len > 0) st_foreach(tbl, ibf_dump_cdhash_i, (VALUE)dump); + break; + } + default: + ibf_dump_object_unsupported(dump, obj); + break; + } +} + +static VALUE +ibf_load_object_imemo(const struct ibf_load *load, const struct ibf_object_header *header, ibf_offset_t offset) +{ + long len = (long)ibf_load_small_value(load, &offset); + VALUE obj = cdhash_new(len); + + int i; + for (i = 0; i < len; i++) { + VALUE key_index = ibf_load_small_value(load, &offset); + VALUE val = ibf_load_small_value(load, &offset); + + VALUE key = ibf_load_object(load, key_index); + cdhash_aset(obj, key, val); + } return obj; } @@ -13401,7 +14447,7 @@ ibf_load_object_struct(const struct ibf_load *load, const struct ibf_object_head VALUE end = ibf_load_object(load, range->end); VALUE obj = rb_range_new(beg, end, range->excl); if (header->internal) rb_obj_hide(obj); - if (header->frozen) rb_obj_freeze(obj); + if (header->frozen) RB_OBJ_SET_FROZEN_SHAREABLE(obj); return obj; } @@ -13429,7 +14475,7 @@ ibf_load_object_bignum(const struct ibf_load *load, const struct ibf_object_head big_unpack_flags | (sign == 0 ? INTEGER_PACK_NEGATIVE : 0)); if (header->internal) rb_obj_hide(obj); - if (header->frozen) rb_obj_freeze(obj); + if (header->frozen) RB_OBJ_SET_FROZEN_SHAREABLE(obj); return obj; } @@ -13490,7 +14536,7 @@ ibf_load_object_complex_rational(const struct ibf_load *load, const struct ibf_o rb_complex_new(a, b) : rb_rational_new(a, b); if (header->internal) rb_obj_hide(obj); - if (header->frozen) rb_obj_freeze(obj); + if (header->frozen) rb_ractor_make_shareable(rb_obj_freeze(obj)); return obj; } @@ -13546,7 +14592,7 @@ static const ibf_dump_object_function dump_object_functions[RUBY_T_MASK+1] = { ibf_dump_object_unsupported, /* 0x17 */ ibf_dump_object_unsupported, /* 0x18 */ ibf_dump_object_unsupported, /* 0x19 */ - ibf_dump_object_unsupported, /* T_IMEMO 0x1a */ + ibf_dump_object_imemo, /* T_IMEMO 0x1a */ ibf_dump_object_unsupported, /* T_NODE 0x1b */ ibf_dump_object_unsupported, /* T_ICLASS 0x1c */ ibf_dump_object_unsupported, /* T_ZOMBIE 0x1d */ @@ -13603,7 +14649,7 @@ ibf_dump_object_object(struct ibf_dump *dump, VALUE obj) else { obj_header.internal = SPECIAL_CONST_P(obj) ? FALSE : (RBASIC_CLASS(obj) == 0) ? TRUE : FALSE; obj_header.special_const = FALSE; - obj_header.frozen = FL_TEST(obj, FL_FREEZE) ? TRUE : FALSE; + obj_header.frozen = OBJ_FROZEN(obj) ? TRUE : FALSE; ibf_dump_object_object_header(dump, obj_header); (*dump_object_functions[obj_header.type])(dump, obj); } @@ -13639,7 +14685,7 @@ static const ibf_load_object_function load_object_functions[RUBY_T_MASK+1] = { ibf_load_object_unsupported, /* 0x17 */ ibf_load_object_unsupported, /* 0x18 */ ibf_load_object_unsupported, /* 0x19 */ - ibf_load_object_unsupported, /* T_IMEMO 0x1a */ + ibf_load_object_imemo, /* T_IMEMO 0x1a */ ibf_load_object_unsupported, /* T_NODE 0x1b */ ibf_load_object_unsupported, /* T_ICLASS 0x1c */ ibf_load_object_unsupported, /* T_ZOMBIE 0x1d */ @@ -13968,6 +15014,8 @@ ibf_load_setup_bytes(struct ibf_load *load, VALUE loader_obj, const char *bytes, static void ibf_load_setup(struct ibf_load *load, VALUE loader_obj, VALUE str) { + StringValue(str); + if (RSTRING_LENINT(str) < (int)sizeof(struct ibf_header)) { rb_raise(rb_eRuntimeError, "broken binary format"); } @@ -13976,7 +15024,7 @@ ibf_load_setup(struct ibf_load *load, VALUE loader_obj, VALUE str) str = rb_str_new(RSTRING_PTR(str), RSTRING_LEN(str)); } - ibf_load_setup_bytes(load, loader_obj, StringValuePtr(str), RSTRING_LEN(str)); + ibf_load_setup_bytes(load, loader_obj, RSTRING_PTR(str), RSTRING_LEN(str)); RB_OBJ_WRITE(loader_obj, &load->str, str); } @@ -13993,7 +15041,7 @@ static void ibf_loader_free(void *ptr) { struct ibf_load *load = (struct ibf_load *)ptr; - ruby_xfree(load); + SIZED_FREE(load); } static size_t |