diff options
Diffstat (limited to 'zjit')
57 files changed, 36874 insertions, 5435 deletions
diff --git a/zjit/.gitignore b/zjit/.gitignore index 2f7896d1d1..100c0dfb5c 100644 --- a/zjit/.gitignore +++ b/zjit/.gitignore @@ -1 +1,2 @@ target/ +*.pending-snap diff --git a/zjit/Cargo.toml b/zjit/Cargo.toml index a86117d6e2..ef656c5252 100644 --- a/zjit/Cargo.toml +++ b/zjit/Cargo.toml @@ -6,15 +6,18 @@ rust-version = "1.85.0" # Minimally supported rust version publish = false # Don't publish to crates.io [dependencies] -# No required dependencies to simplify build process. TODO: Link to yet to be -# written rationale. Optional For development and testing purposes +# No required dependencies to simplify build process. +# Optional For development and testing purposes. capstone = { version = "0.13.0", optional = true } +jit = { version = "0.1.0", path = "../jit" } [dev-dependencies] -expect-test = "1.5.1" +insta = "1.43.1" # NOTE: Development builds select a set of these via configure.ac # For debugging, `make V=1` shows exact cargo invocation. [features] # Support --yjit-dump-disasm and RubyVM::YJIT.disasm using libcapstone. disasm = ["capstone"] +runtime_checks = [] +stats_allocator = [] diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 8162f9a9ed..794293d1d3 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -69,32 +69,22 @@ fn main() { // Import YARV bytecode instruction constants .allowlist_type("ruby_vminsn_type") - // From include/ruby/internal/special_consts.h .allowlist_type("ruby_special_consts") - - // From include/ruby/internal/intern/string.h .allowlist_function("rb_utf8_str_new") .allowlist_function("rb_str_buf_append") .allowlist_function("rb_str_dup") - - // From encindex.h + .allowlist_function("rb_str_getbyte") .allowlist_type("ruby_preserved_encindex") - - // From include/ruby/ruby.h .allowlist_function("rb_class2name") // This struct is public to Ruby C extensions - // From include/ruby/internal/core/rbasic.h .allowlist_type("RBasic") - // From include/ruby/internal/core/rstring.h .allowlist_type("ruby_rstring_flags") - // From internal.h // This function prints info about a value and is useful for debugging - .allowlist_function("rb_obj_info_dump") + .allowlist_function("rb_raw_obj_info") - // For testing .allowlist_function("ruby_init") .allowlist_function("ruby_init_stack") .allowlist_function("ruby_options") @@ -106,72 +96,57 @@ fn main() { // For crashing .allowlist_function("rb_bug") - // From shape.h .allowlist_function("rb_obj_shape_id") - .allowlist_function("rb_shape_lookup") .allowlist_function("rb_shape_id_offset") .allowlist_function("rb_shape_get_iv_index") .allowlist_function("rb_shape_transition_add_ivar_no_warnings") - .allowlist_function("rb_shape_id") - .allowlist_function("rb_shape_obj_too_complex_p") + .allowlist_function("rb_jit_shape_capacity") + .allowlist_var("rb_invalid_shape_id") + .allowlist_type("shape_id_fl_type") + .allowlist_var("VM_KW_SPECIFIED_BITS_MAX") .allowlist_var("SHAPE_ID_NUM_BITS") - - // From ruby/internal/intern/object.h .allowlist_function("rb_obj_is_kind_of") .allowlist_function("rb_obj_frozen_p") .allowlist_function("rb_class_inherited_p") - - // From ruby/internal/encoding/encoding.h + .allowlist_function("rb_class_real") .allowlist_type("ruby_encoding_consts") - - // From include/hash.h .allowlist_function("rb_hash_new") - - // From internal/hash.h .allowlist_function("rb_hash_new_with_size") .allowlist_function("rb_hash_resurrect") .allowlist_function("rb_hash_stlike_foreach") .allowlist_function("rb_to_hash_type") - - // From include/ruby/st.h .allowlist_type("st_retval") - - // From include/ruby/internal/intern/hash.h .allowlist_function("rb_hash_aset") .allowlist_function("rb_hash_aref") .allowlist_function("rb_hash_bulk_insert") .allowlist_function("rb_hash_stlike_lookup") - - // From include/ruby/internal/intern/array.h .allowlist_function("rb_ary_new_capa") .allowlist_function("rb_ary_store") .allowlist_function("rb_ary_resurrect") .allowlist_function("rb_ary_cat") + .allowlist_function("rb_ary_concat") .allowlist_function("rb_ary_clear") .allowlist_function("rb_ary_dup") .allowlist_function("rb_ary_push") + .allowlist_function("rb_ary_pop") .allowlist_function("rb_ary_unshift_m") - - // From internal/array.h .allowlist_function("rb_ec_ary_new_from_values") .allowlist_function("rb_ary_tmp_new_from_values") - - // From include/ruby/internal/intern/class.h + .allowlist_function("rb_ary_entry") .allowlist_function("rb_class_attached_object") .allowlist_function("rb_singleton_class") .allowlist_function("rb_define_class") - - // From include/ruby/internal/core/rclass.h .allowlist_function("rb_class_get_superclass") - - // From include/ruby/internal/gc.h + .allowlist_function("rb_gc_disable") + .allowlist_function("rb_gc_enable") .allowlist_function("rb_gc_mark") .allowlist_function("rb_gc_mark_movable") .allowlist_function("rb_gc_location") .allowlist_function("rb_gc_writebarrier") + .allowlist_function("rb_gc_writebarrier_remember") + .allowlist_function("rb_zjit_writebarrier_check_immediate") // VALUE variables for Ruby class objects - // From include/ruby/internal/globals.h .allowlist_var("rb_cBasicObject") .allowlist_var("rb_cObject") .allowlist_var("rb_cModule") @@ -183,91 +158,69 @@ fn main() { .allowlist_var("rb_cSymbol") .allowlist_var("rb_cFloat") .allowlist_var("rb_cNumeric") + .allowlist_var("rb_cRange") .allowlist_var("rb_cString") .allowlist_var("rb_cThread") .allowlist_var("rb_cArray") .allowlist_var("rb_cHash") + .allowlist_var("rb_cSet") .allowlist_var("rb_cClass") + .allowlist_var("rb_cRegexp") .allowlist_var("rb_cISeq") - // From include/ruby/internal/fl_type.h .allowlist_type("ruby_fl_type") .allowlist_type("ruby_fl_ushift") - - // From include/ruby/internal/core/robject.h .allowlist_type("ruby_robject_flags") - - // From include/ruby/internal/core/rarray.h .allowlist_type("ruby_rarray_flags") .allowlist_type("ruby_rarray_consts") - - // From include/ruby/internal/core/rclass.h .allowlist_type("ruby_rmodule_flags") - - // From ruby/internal/globals.h .allowlist_var("rb_mKernel") - - // From vm_callinfo.h .allowlist_type("vm_call_flag_bits") .allowlist_type("rb_call_data") .blocklist_type("rb_callcache.*") // Not used yet - opaque to make it easy to import rb_call_data .opaque_type("rb_callcache.*") .allowlist_type("rb_callinfo") - - // From vm_insnhelper.h .allowlist_var("VM_ENV_DATA_INDEX_ME_CREF") .allowlist_var("rb_block_param_proxy") - - // From include/ruby/internal/intern/range.h .allowlist_function("rb_range_new") - - // From include/ruby/internal/symbol.h .allowlist_function("rb_intern") .allowlist_function("rb_intern2") .allowlist_function("rb_id2sym") .allowlist_function("rb_sym2id") .allowlist_function("rb_str_intern") .allowlist_function("rb_id2str") - - // From internal/numeric.h + .allowlist_function("rb_sym2str") .allowlist_function("rb_fix_aref") .allowlist_function("rb_float_plus") .allowlist_function("rb_float_minus") .allowlist_function("rb_float_mul") .allowlist_function("rb_float_div") - - // From internal/string.h .allowlist_type("ruby_rstring_private_flags") .allowlist_function("rb_ec_str_resurrect") .allowlist_function("rb_str_concat_literals") .allowlist_function("rb_obj_as_string_result") .allowlist_function("rb_str_byte_substr") .allowlist_function("rb_str_substr_two_fixnums") - - // From include/ruby/internal/intern/parse.h .allowlist_function("rb_backref_get") - - // From include/ruby/internal/intern/re.h .allowlist_function("rb_reg_last_match") .allowlist_function("rb_reg_match_pre") .allowlist_function("rb_reg_match_post") .allowlist_function("rb_reg_match_last") .allowlist_function("rb_reg_nth_match") - - // From internal/re.h .allowlist_function("rb_reg_new_ary") + .allowlist_var("ARG_ENCODING_FIXED") + .allowlist_var("ARG_ENCODING_NONE") + .allowlist_var("ONIG_OPTION_IGNORECASE") + .allowlist_var("ONIG_OPTION_EXTEND") + .allowlist_var("ONIG_OPTION_MULTILINE") // `ruby_value_type` is a C enum and this stops it from // prefixing all the members with the name of the type .prepend_enum_name(false) .translate_enum_integer_types(true) // so we get fixed width Rust types for members - // From include/ruby/internal/value_type.h .allowlist_type("ruby_value_type") // really old C extension API - // From include/ruby/internal/hash.h .allowlist_type("ruby_rhash_flags") // really old C extension API - - // From method.h .allowlist_type("rb_method_visibility_t") .allowlist_type("rb_method_type_t") .allowlist_type("method_optimized_type") @@ -278,11 +231,7 @@ fn main() { .blocklist_type("rb_method_cfunc_t") .blocklist_type("rb_method_definition_.*") // Large struct with a bitfield and union of many types - don't import (yet?) .opaque_type("rb_method_definition_.*") - - // From numeric.c .allowlist_function("rb_float_new") - - // From vm_core.h .allowlist_var("rb_mRubyVMFrozenCore") .allowlist_var("VM_BLOCK_HANDLER_NONE") .allowlist_type("vm_frame_env_flags") @@ -320,87 +269,91 @@ fn main() { .allowlist_type("vm_check_match_type") .allowlist_type("vm_opt_newarray_send_type") .allowlist_type("rb_iseq_type") - - // From zjit.c + .allowlist_type("rb_event_flag_t") .allowlist_function("rb_object_shape_count") .allowlist_function("rb_iseq_(get|set)_zjit_payload") .allowlist_function("rb_iseq_pc_at_idx") .allowlist_function("rb_iseq_opcode_at_pc") - .allowlist_function("rb_zjit_reserve_addr_space") - .allowlist_function("rb_zjit_mark_writable") - .allowlist_function("rb_zjit_mark_executable") - .allowlist_function("rb_zjit_mark_unused") - .allowlist_function("rb_zjit_get_page_size") - .allowlist_function("rb_zjit_iseq_builtin_attrs") + .allowlist_function("rb_jit_reserve_addr_space") + .allowlist_function("rb_jit_mark_writable") + .allowlist_function("rb_jit_mark_executable") + .allowlist_function("rb_jit_mark_unused") + .allowlist_function("rb_jit_get_page_size") + .allowlist_function("rb_jit_array_len") + .allowlist_function("rb_jit_fix_div_fix") + .allowlist_function("rb_jit_iseq_builtin_attrs") + .allowlist_function("rb_jit_str_concat_codepoint") .allowlist_function("rb_zjit_iseq_inspect") + .allowlist_function("rb_zjit_iseq_insn_set") + .allowlist_function("rb_zjit_local_id") .allowlist_function("rb_set_cfp_(pc|sp)") .allowlist_function("rb_c_method_tracing_currently_enabled") + .allowlist_function("rb_zjit_method_tracing_currently_enabled") .allowlist_function("rb_full_cfunc_return") - .allowlist_function("rb_zjit_vm_lock_then_barrier") - .allowlist_function("rb_zjit_vm_unlock") .allowlist_function("rb_assert_(iseq|cme)_handle") .allowlist_function("rb_IMEMO_TYPE_P") - .allowlist_function("rb_iseq_reset_jit_func") .allowlist_function("rb_RSTRING_PTR") .allowlist_function("rb_RSTRING_LEN") .allowlist_function("rb_ENCODING_GET") + .allowlist_function("rb_zjit_exit_locations_dict") .allowlist_function("rb_optimized_call") - .allowlist_function("rb_zjit_icache_invalidate") + .allowlist_function("rb_jit_icache_invalidate") .allowlist_function("rb_zjit_print_exception") - .allowlist_type("robject_offsets") - .allowlist_type("rstring_offsets") - - // from vm_sync.h + .allowlist_function("rb_zjit_singleton_class_p") + .allowlist_function("rb_zjit_defined_ivar") + .allowlist_function("rb_zjit_insn_leaf") + .allowlist_type("jit_bindgen_constants") + .allowlist_type("zjit_struct_offsets") + .allowlist_function("rb_assert_holding_vm_lock") + .allowlist_function("rb_jit_shape_too_complex_p") + .allowlist_function("rb_jit_multi_ractor_p") + .allowlist_function("rb_jit_vm_lock_then_barrier") + .allowlist_function("rb_jit_vm_unlock") + .allowlist_function("rb_jit_for_each_iseq") + .allowlist_function("rb_iseq_reset_jit_func") .allowlist_function("rb_vm_barrier") // Not sure why it's picking these up, but don't. .blocklist_type("FILE") .blocklist_type("_IO_.*") - // From internal/compile.h .allowlist_function("rb_vm_insn_decode") - - // from internal/cont.h .allowlist_function("rb_jit_cont_each_iseq") - - // From iseq.h .allowlist_function("rb_vm_insn_addr2opcode") .allowlist_function("rb_iseqw_to_iseq") .allowlist_function("rb_iseq_label") .allowlist_function("rb_iseq_line_no") + .allowlist_function("rb_iseq_defined_string") .allowlist_type("defined_type") - - // From builtin.h .allowlist_type("rb_builtin_function.*") - - // From internal/variable.h .allowlist_function("rb_gvar_(get|set)") .allowlist_function("rb_ensure_iv_list_size") - - // From include/ruby/internal/intern/variable.h .allowlist_function("rb_attr_get") .allowlist_function("rb_ivar_defined") .allowlist_function("rb_ivar_get") + .allowlist_function("rb_ivar_get_at_no_ractor_check") + .allowlist_function("rb_ivar_set") .allowlist_function("rb_mod_name") - - // From include/ruby/internal/intern/vm.h + .allowlist_var("rb_vm_insn_count") .allowlist_function("rb_get_alloc_func") - - // From internal/object.h .allowlist_function("rb_class_allocate_instance") .allowlist_function("rb_obj_equal") - - // From gc.h and internal/gc.h + .allowlist_function("rb_class_new_instance_pass_kw") + .allowlist_function("rb_obj_alloc") .allowlist_function("rb_obj_info") - .allowlist_function("ruby_xfree") - + .allowlist_function("rb_obj_frozen_p") // From include/ruby/debug.h .allowlist_function("rb_profile_frames") + .allowlist_function("ruby_xfree") + .allowlist_function("rb_profile_frames") // Functions used for code generation .allowlist_function("rb_insn_name") .allowlist_function("rb_insn_len") .allowlist_function("rb_yarv_class_of") + .allowlist_function("rb_zjit_class_initialized_p") + .allowlist_function("rb_zjit_class_has_default_allocator") + .allowlist_function("rb_zjit_class_get_alloc_func") .allowlist_function("rb_get_ec_cfp") .allowlist_function("rb_get_cfp_iseq") .allowlist_function("rb_get_cfp_pc") @@ -409,8 +362,9 @@ fn main() { .allowlist_function("rb_get_cfp_ep") .allowlist_function("rb_get_cfp_ep_level") .allowlist_function("rb_get_cme_def_type") - .allowlist_function("rb_zjit_multi_ractor_p") .allowlist_function("rb_zjit_constcache_shareable") + .allowlist_function("rb_zjit_vm_search_method") + .allowlist_function("rb_zjit_cme_is_cfunc") .allowlist_function("rb_get_cme_def_body_attr_id") .allowlist_function("rb_get_symbol_id") .allowlist_function("rb_get_cme_def_body_optimized_type") @@ -422,6 +376,7 @@ fn main() { .allowlist_function("rb_get_mct_func") .allowlist_function("rb_get_def_iseq_ptr") .allowlist_function("rb_get_def_bmethod_proc") + .allowlist_function("rb_jit_get_proc_ptr") .allowlist_function("rb_iseq_encoded_size") .allowlist_function("rb_get_iseq_body_total_calls") .allowlist_function("rb_get_iseq_body_local_iseq") @@ -453,12 +408,11 @@ fn main() { .allowlist_function("rb_yarv_str_eql_internal") .allowlist_function("rb_str_neq_internal") .allowlist_function("rb_yarv_ary_entry_internal") + .allowlist_function("rb_vm_get_untagged_block_handler") .allowlist_function("rb_FL_TEST") .allowlist_function("rb_FL_TEST_RAW") .allowlist_function("rb_RB_TYPE_P") .allowlist_function("rb_BASIC_OP_UNREDEFINED_P") - .allowlist_function("rb_RSTRUCT_LEN") - .allowlist_function("rb_RSTRUCT_SET") .allowlist_function("rb_vm_ci_argc") .allowlist_function("rb_vm_ci_mid") .allowlist_function("rb_vm_ci_flag") @@ -475,10 +429,14 @@ fn main() { // We define these manually, don't import them .blocklist_type("VALUE") .blocklist_type("ID") - - // From iseq.h - .opaque_type("rb_iseq_t") - .blocklist_type("rb_iseq_t") + .blocklist_type("rb_iseq_constant_body") + + // Avoid binding to stuff we don't use + .blocklist_item("rb_thread_struct.*") + .opaque_type("rb_thread_struct.*") + .blocklist_item("iseq_inline_storage_entry_.*") + .opaque_type("iseq_inline_storage_entry") + .opaque_type("iseq_compile_data") // Finish the builder and generate the bindings. .generate() diff --git a/zjit/build.rs b/zjit/build.rs index 6aec5407f6..4ee3d65b33 100644 --- a/zjit/build.rs +++ b/zjit/build.rs @@ -5,9 +5,11 @@ fn main() { // option_env! automatically registers a rerun-if-env-changed if let Some(ruby_build_dir) = option_env!("RUBY_BUILD_DIR") { - // Link against libminiruby + // Link against libminiruby.a println!("cargo:rustc-link-search=native={ruby_build_dir}"); println!("cargo:rustc-link-lib=static:-bundle=miniruby"); + // Re-link when libminiruby.a changes + println!("cargo:rerun-if-changed={ruby_build_dir}/libminiruby.a"); // System libraries that libminiruby needs. Has to be // ordered after -lminiruby above. diff --git a/zjit/src/asm/arm64/arg/mod.rs b/zjit/src/asm/arm64/arg/mod.rs index 9aff99a817..7eb37834f9 100644 --- a/zjit/src/asm/arm64/arg/mod.rs +++ b/zjit/src/asm/arm64/arg/mod.rs @@ -10,7 +10,6 @@ mod sys_reg; mod truncate; pub use bitmask_imm::BitmaskImmediate; -#[cfg(target_arch = "aarch64")] pub use condition::Condition; pub use inst_offset::InstructionOffset; pub use sf::Sf; diff --git a/zjit/src/asm/arm64/arg/sf.rs b/zjit/src/asm/arm64/arg/sf.rs index c2fd33302c..b6091821e9 100644 --- a/zjit/src/asm/arm64/arg/sf.rs +++ b/zjit/src/asm/arm64/arg/sf.rs @@ -13,7 +13,7 @@ impl From<u8> for Sf { match num_bits { 64 => Sf::Sf64, 32 => Sf::Sf32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/arg/shifted_imm.rs b/zjit/src/asm/arm64/arg/shifted_imm.rs index 4602ac64ab..06daefdef7 100644 --- a/zjit/src/asm/arm64/arg/shifted_imm.rs +++ b/zjit/src/asm/arm64/arg/shifted_imm.rs @@ -16,7 +16,6 @@ pub struct ShiftedImmediate { impl TryFrom<u64> for ShiftedImmediate { type Error = (); - /// Attempt to convert a u64 into a BitmaskImm. fn try_from(value: u64) -> Result<Self, Self::Error> { let current = value; if current < 2_u64.pow(12) { diff --git a/zjit/src/asm/arm64/inst/atomic.rs b/zjit/src/asm/arm64/inst/atomic.rs index dce9affedf..0917a4fd1c 100644 --- a/zjit/src/asm/arm64/inst/atomic.rs +++ b/zjit/src/asm/arm64/inst/atomic.rs @@ -14,7 +14,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/branch.rs b/zjit/src/asm/arm64/inst/branch.rs index 14fcb2e9fd..2db52e5d31 100644 --- a/zjit/src/asm/arm64/inst/branch.rs +++ b/zjit/src/asm/arm64/inst/branch.rs @@ -89,7 +89,7 @@ mod tests { #[test] fn test_ret() { let result: u32 = Branch::ret(30).into(); - assert_eq!(0xd65f03C0, result); + assert_eq!(0xd65f03c0, result); } #[test] diff --git a/zjit/src/asm/arm64/inst/branch_cond.rs b/zjit/src/asm/arm64/inst/branch_cond.rs index 49f44c1668..266e9ccb31 100644 --- a/zjit/src/asm/arm64/inst/branch_cond.rs +++ b/zjit/src/asm/arm64/inst/branch_cond.rs @@ -47,7 +47,6 @@ impl From<BranchCond> for [u8; 4] { } } -/* #[cfg(test)] mod tests { use super::*; @@ -77,4 +76,3 @@ mod tests { assert_eq!(0x54800000, result); } } -*/ diff --git a/zjit/src/asm/arm64/inst/conditional.rs b/zjit/src/asm/arm64/inst/conditional.rs index 90b5993cad..1e26c7408b 100644 --- a/zjit/src/asm/arm64/inst/conditional.rs +++ b/zjit/src/asm/arm64/inst/conditional.rs @@ -60,7 +60,6 @@ impl From<Conditional> for [u8; 4] { } } -/* #[cfg(test)] mod tests { use super::*; @@ -72,4 +71,3 @@ mod tests { assert_eq!(0x9a821020, result); } } -*/ diff --git a/zjit/src/asm/arm64/inst/load_literal.rs b/zjit/src/asm/arm64/inst/load_literal.rs index 817e893553..37b5f3c7a7 100644 --- a/zjit/src/asm/arm64/inst/load_literal.rs +++ b/zjit/src/asm/arm64/inst/load_literal.rs @@ -1,3 +1,5 @@ +#![allow(clippy::identity_op)] + use super::super::arg::{InstructionOffset, truncate_imm}; /// The size of the operands being operated on. @@ -13,7 +15,7 @@ impl From<u8> for Opc { match num_bits { 64 => Opc::Size64, 32 => Opc::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/load_register.rs b/zjit/src/asm/arm64/inst/load_register.rs index 3d94e8da1f..80813ffc87 100644 --- a/zjit/src/asm/arm64/inst/load_register.rs +++ b/zjit/src/asm/arm64/inst/load_register.rs @@ -25,7 +25,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/load_store.rs b/zjit/src/asm/arm64/inst/load_store.rs index e27909ae35..d38e851ed7 100644 --- a/zjit/src/asm/arm64/inst/load_store.rs +++ b/zjit/src/asm/arm64/inst/load_store.rs @@ -15,7 +15,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } @@ -124,6 +124,12 @@ impl LoadStore { pub fn sturh(rt: u8, rn: u8, imm9: i16) -> Self { Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size16 } } + + /// STURB (store register, byte, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STURH--Store-Register-Halfword--unscaled--?lang=en> + pub fn sturb(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size8 } + } } /// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> diff --git a/zjit/src/asm/arm64/inst/load_store_exclusive.rs b/zjit/src/asm/arm64/inst/load_store_exclusive.rs index 1106b4cb37..30cb663bdb 100644 --- a/zjit/src/asm/arm64/inst/load_store_exclusive.rs +++ b/zjit/src/asm/arm64/inst/load_store_exclusive.rs @@ -17,7 +17,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/mov.rs b/zjit/src/asm/arm64/inst/mov.rs index eae4565c3a..e9f9091713 100644 --- a/zjit/src/asm/arm64/inst/mov.rs +++ b/zjit/src/asm/arm64/inst/mov.rs @@ -2,6 +2,9 @@ use super::super::arg::Sf; /// Which operation is being performed. enum Op { + /// A movn operation which inverts the immediate and zeroes out the other bits. + MOVN = 0b00, + /// A movz operation which zeroes out the other bits. MOVZ = 0b10, @@ -24,7 +27,7 @@ impl From<u8> for Hw { 16 => Hw::LSL16, 32 => Hw::LSL32, 48 => Hw::LSL48, - _ => panic!("Invalid value for shift: {}", shift) + _ => panic!("Invalid value for shift: {shift}"), } } } @@ -61,6 +64,12 @@ impl Mov { Self { rd, imm16, hw: hw.into(), op: Op::MOVK, sf: num_bits.into() } } + /// MOVN + /// <https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/MOVN--Move-wide-with-NOT-> + pub fn movn(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { + Self { rd, imm16, hw: hw.into(), op: Op::MOVN, sf: num_bits.into() } + } + /// MOVZ /// <https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/MOVZ--Move-wide-with-zero-?lang=en> pub fn movz(rd: u8, imm16: u16, hw: u8, num_bits: u8) -> Self { @@ -105,17 +114,45 @@ mod tests { } #[test] + fn test_movn_unshifted() { + let inst = Mov::movn(0, 123, 0, 64); + let result: u32 = inst.into(); + assert_eq!(0x92800f60, result); + } + + #[test] + fn test_movn_shifted_16() { + let inst = Mov::movn(0, 123, 16, 64); + let result: u32 = inst.into(); + assert_eq!(0x92a00f60, result); + } + + #[test] + fn test_movn_shifted_32() { + let inst = Mov::movn(0, 123, 32, 64); + let result: u32 = inst.into(); + assert_eq!(0x92c00f60, result); + } + + #[test] + fn test_movn_shifted_48() { + let inst = Mov::movn(0, 123, 48, 64); + let result: u32 = inst.into(); + assert_eq!(0x92e00f60, result); + } + + #[test] fn test_movk_shifted_16() { let inst = Mov::movk(0, 123, 16, 64); let result: u32 = inst.into(); - assert_eq!(0xf2A00f60, result); + assert_eq!(0xf2a00f60, result); } #[test] fn test_movk_shifted_32() { let inst = Mov::movk(0, 123, 32, 64); let result: u32 = inst.into(); - assert_eq!(0xf2C00f60, result); + assert_eq!(0xf2c00f60, result); } #[test] diff --git a/zjit/src/asm/arm64/inst/reg_pair.rs b/zjit/src/asm/arm64/inst/reg_pair.rs index 9bffcd8479..39a44c2416 100644 --- a/zjit/src/asm/arm64/inst/reg_pair.rs +++ b/zjit/src/asm/arm64/inst/reg_pair.rs @@ -26,7 +26,7 @@ impl From<u8> for Opc { match num_bits { 64 => Opc::Opc64, 32 => Opc::Opc32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/test_bit.rs b/zjit/src/asm/arm64/inst/test_bit.rs index f7aeca70fd..45f0c2317e 100644 --- a/zjit/src/asm/arm64/inst/test_bit.rs +++ b/zjit/src/asm/arm64/inst/test_bit.rs @@ -17,7 +17,7 @@ impl From<u8> for B5 { match bit_num { 0..=31 => B5::B532, 32..=63 => B5::B564, - _ => panic!("Invalid bit number: {}", bit_num) + _ => panic!("Invalid bit number: {bit_num}"), } } } diff --git a/zjit/src/asm/arm64/mod.rs b/zjit/src/asm/arm64/mod.rs index a5d73d71a5..c30520cd69 100644 --- a/zjit/src/asm/arm64/mod.rs +++ b/zjit/src/asm/arm64/mod.rs @@ -1,4 +1,8 @@ #![allow(dead_code)] // For instructions and operands we're not currently using. +#![allow(clippy::upper_case_acronyms)] +#![allow(clippy::identity_op)] +#![allow(clippy::self_named_constructors)] +#![allow(clippy::unusual_byte_groupings)] use crate::asm::CodeBlock; @@ -13,6 +17,21 @@ use inst::*; pub use arg::*; pub use opnd::*; +/// The extend type for register operands in extended register instructions. +/// It's the result size is determined by the destination register and +/// the source size interpreted using the last letter. +#[derive(Clone, Copy)] +pub enum ExtendType { + UXTB = 0b000, // unsigned extend byte + UXTH = 0b001, // unsigned extend halfword + UXTW = 0b010, // unsigned extend word + UXTX = 0b011, // unsigned extend doubleword + SXTB = 0b100, // signed extend byte + SXTH = 0b101, // signed extend halfword + SXTW = 0b110, // signed extend word + SXTX = 0b111, // signed extend doubleword +} + /// Checks that a signed value fits within the specified number of bits. pub const fn imm_fits_bits(imm: i64, num_bits: u8) -> bool { let minimum = if num_bits == 64 { i64::MIN } else { -(2_i64.pow((num_bits as u32) - 1)) }; @@ -59,6 +78,42 @@ pub fn add(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { cb.write_bytes(&bytes); } +/// Encode ADD (extended register) +/// +/// <https://developer.arm.com/documentation/ddi0602/2023-09/Base-Instructions/ADD--extended-register---Add--extended-register--> +/// +/// 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 09 08 07 06 05 04 03 02 01 00 +/// 0 1 0 1 1 0 0 1 │ │ │ │ │ │ │ │ │ │ +/// sf op S └────rm─────┘ └option┘ └─imm3─┘ └────rn─────┘ └────rd─────┘ +fn encode_add_extend(rd: u8, rn: u8, rm: u8, extend_type: ExtendType, shift: u8, num_bits: u8) -> [u8; 4] { + assert!(shift <= 4, "shift must be 0-4"); + + ((Sf::from(num_bits) as u32) << 31 | + 0b0 << 30 | // op = 0 for add + 0b0 << 29 | // S = 0 for non-flag-setting + 0b01011001 << 21 | + (rm as u32) << 16 | + (extend_type as u32) << 13 | + (shift as u32) << 10 | + (rn as u32) << 5 | + rd as u32).to_le_bytes() +} + +/// ADD (extended register) - add rn and rm with UXTX extension (no extension for 64-bit registers) +/// This is equivalent to a regular ADD for 64-bit registers since UXTX with shift 0 means no modification. +/// For reg_no=31, rd and rn mean SP while with rm means the zero register. +pub fn add_extended(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { + let bytes: [u8; 4] = match (rd, rn, rm) { + (A64Opnd::Reg(rd), A64Opnd::Reg(rn), A64Opnd::Reg(rm)) => { + assert!(rd.num_bits == rn.num_bits, "rd and rn must be of the same size."); + encode_add_extend(rd.reg_no, rn.reg_no, rm.reg_no, ExtendType::UXTX, 0, rd.num_bits) + }, + _ => panic!("Invalid operand combination to add_extend instruction."), + }; + + cb.write_bytes(&bytes); +} + /// ADDS - add rn and rm, put the result in rd, update flags pub fn adds(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, rm: A64Opnd) { let bytes: [u8; 4] = match (rd, rn, rm) { @@ -186,7 +241,7 @@ pub fn asr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { SBFM::asr(rd.reg_no, rn.reg_no, shift.try_into().unwrap(), rd.num_bits).into() }, - _ => panic!("Invalid operand combination to asr instruction: asr {:?}, {:?}, {:?}", rd, rn, shift), + _ => panic!("Invalid operand combination to asr instruction: asr {rd:?}, {rn:?}, {shift:?}"), }; cb.write_bytes(&bytes); @@ -217,6 +272,7 @@ pub const fn bcond_offset_fits_bits(offset: i64) -> bool { /// B.cond - branch to target if condition is true pub fn bcond(cb: &mut CodeBlock, cond: u8, offset: InstructionOffset) { + _ = Condition; assert!(bcond_offset_fits_bits(offset.into()), "The offset must be 19 bits or less."); let bytes: [u8; 4] = BranchCond::bcond(cond, offset).into(); @@ -593,7 +649,7 @@ pub fn lsl(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { ShiftImm::lsl(rd.reg_no, rn.reg_no, uimm as u8, rd.num_bits).into() }, - _ => panic!("Invalid operands combination to lsl instruction") + _ => panic!("Invalid operands combination {rd:?} {rn:?} {shift:?} to lsl instruction") }; cb.write_bytes(&bytes); @@ -644,7 +700,7 @@ pub fn mov(cb: &mut CodeBlock, rd: A64Opnd, rm: A64Opnd) { LogicalImm::mov(rd.reg_no, bitmask_imm, rd.num_bits).into() }, - _ => panic!("Invalid operand combination to mov instruction") + _ => panic!("Invalid operand combination to mov instruction: {rd:?}, {rm:?}") }; cb.write_bytes(&bytes); @@ -664,6 +720,21 @@ pub fn movk(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { cb.write_bytes(&bytes); } +/// MOVN - load a register with the complement of a shifted then zero extended 16-bit immediate +/// <https://developer.arm.com/documentation/ddi0602/2025-06/Base-Instructions/MOVN--Move-wide-with-NOT-> +pub fn movn(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { + let bytes: [u8; 4] = match (rd, imm16) { + (A64Opnd::Reg(rd), A64Opnd::UImm(imm16)) => { + assert!(uimm_fits_bits(imm16, 16), "The immediate operand must be 16 bits or less."); + + Mov::movn(rd.reg_no, imm16 as u16, shift, rd.num_bits).into() + }, + _ => panic!("Invalid operand combination to movn instruction.") + }; + + cb.write_bytes(&bytes); +} + /// MOVZ - move a 16 bit immediate into a register, zero the other bits pub fn movz(cb: &mut CodeBlock, rd: A64Opnd, imm16: A64Opnd, shift: u8) { let bytes: [u8; 4] = match (rd, imm16) { @@ -936,11 +1007,11 @@ pub fn stur(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { let bytes: [u8; 4] = match (rt, rn) { (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { assert!(rn.num_bits == 32 || rn.num_bits == 64); - assert!(mem_disp_fits_bits(rn.disp), "Expected displacement to be 9 bits or less"); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement {} to be 9 bits or less", rn.disp); LoadStore::stur(rt.reg_no, rn.base_reg_no, rn.disp as i16, rn.num_bits).into() }, - _ => panic!("Invalid operand combination to stur instruction.") + _ => panic!("Invalid operand combination to stur instruction: {rt:?}, {rn:?}") }; cb.write_bytes(&bytes); @@ -955,7 +1026,21 @@ pub fn sturh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { LoadStore::sturh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() }, - _ => panic!("Invalid operand combination to stur instruction.") + _ => panic!("Invalid operand combination to sturh instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +pub fn sturb(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 8); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement {} to be 9 bits or less", rn.disp); + + LoadStore::sturb(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to sturb instruction: {rt:?}, {rn:?}") }; cb.write_bytes(&bytes); @@ -1138,16 +1223,16 @@ fn cbz_cbnz(num_bits: u8, op: bool, offset: InstructionOffset, rt: u8) -> [u8; 4 rt as u32).to_le_bytes() } -/* #[cfg(test)] mod tests { use super::*; + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; - /// Check that the bytes for an instruction sequence match a hex string - fn check_bytes<R>(bytes: &str, run: R) where R: FnOnce(&mut super::CodeBlock) { - let mut cb = super::CodeBlock::new_dummy(128); + fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); run(&mut cb); - assert_eq!(format!("{:x}", cb), bytes); + cb } #[test] @@ -1175,94 +1260,130 @@ mod tests { #[test] fn test_add_reg() { - check_bytes("2000028b", |cb| add(cb, X0, X1, X2)); + let cb = compile(|cb| add(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028b"); } #[test] fn test_add_uimm() { - check_bytes("201c0091", |cb| add(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); } #[test] fn test_add_imm_positive() { - check_bytes("201c0091", |cb| add(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); } #[test] fn test_add_imm_negative() { - check_bytes("201c00d1", |cb| add(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); } #[test] fn test_adds_reg() { - check_bytes("200002ab", |cb| adds(cb, X0, X1, X2)); + let cb = compile(|cb| adds(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ab"); } #[test] fn test_adds_uimm() { - check_bytes("201c00b1", |cb| adds(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); } #[test] fn test_adds_imm_positive() { - check_bytes("201c00b1", |cb| adds(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); } #[test] fn test_adds_imm_negative() { - check_bytes("201c00f1", |cb| adds(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); } #[test] fn test_adr() { - check_bytes("aa000010", |cb| adr(cb, X10, A64Opnd::new_imm(20))); + let cb = compile(|cb| adr(cb, X10, A64Opnd::new_imm(20))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adr x10, #0x14"); + assert_snapshot!(cb.hexdump(), @"aa000010"); } #[test] fn test_adrp() { - check_bytes("4a000090", |cb| adrp(cb, X10, A64Opnd::new_imm(0x8000))); + let cb = compile(|cb| adrp(cb, X10, A64Opnd::new_imm(0x8000))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adrp x10, #0x8000"); + assert_snapshot!(cb.hexdump(), @"4a000090"); } #[test] fn test_and_register() { - check_bytes("2000028a", |cb| and(cb, X0, X1, X2)); + let cb = compile(|cb| and(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028a"); } #[test] fn test_and_immediate() { - check_bytes("20084092", |cb| and(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| and(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"20084092"); } #[test] fn test_and_32b_immediate() { - check_bytes("404c0012", |cb| and(cb, W0, W2, A64Opnd::new_uimm(0xfffff))); + let cb = compile(|cb| and(cb, W0, W2, A64Opnd::new_uimm(0xfffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and w0, w2, #0xfffff"); + assert_snapshot!(cb.hexdump(), @"404c0012"); } #[test] fn test_ands_register() { - check_bytes("200002ea", |cb| ands(cb, X0, X1, X2)); + let cb = compile(|cb| ands(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ea"); } #[test] fn test_ands_immediate() { - check_bytes("200840f2", |cb| ands(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| ands(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"200840f2"); } #[test] fn test_asr() { - check_bytes("b4fe4a93", |cb| asr(cb, X20, X21, A64Opnd::new_uimm(10))); + let cb = compile(|cb| asr(cb, X20, X21, A64Opnd::new_uimm(10))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: asr x20, x21, #0xa"); + assert_snapshot!(cb.hexdump(), @"b4fe4a93"); } #[test] fn test_bcond() { let offset = InstructionOffset::from_insns(0x100); - check_bytes("01200054", |cb| bcond(cb, Condition::NE, offset)); + let cb = compile(|cb| bcond(cb, Condition::NE, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b.ne #0x400"); + assert_snapshot!(cb.hexdump(), @"01200054"); } #[test] fn test_b() { let offset = InstructionOffset::from_insns((1 << 25) - 1); - check_bytes("ffffff15", |cb| b(cb, offset)); + let cb = compile(|cb| b(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b #0x7fffffc"); + assert_snapshot!(cb.hexdump(), @"ffffff15"); } #[test] @@ -1270,7 +1391,7 @@ mod tests { fn test_b_too_big() { // There are 26 bits available let offset = InstructionOffset::from_insns(1 << 25); - check_bytes("", |cb| b(cb, offset)); + compile(|cb| b(cb, offset)); } #[test] @@ -1278,13 +1399,15 @@ mod tests { fn test_b_too_small() { // There are 26 bits available let offset = InstructionOffset::from_insns(-(1 << 25) - 1); - check_bytes("", |cb| b(cb, offset)); + compile(|cb| b(cb, offset)); } #[test] fn test_bl() { let offset = InstructionOffset::from_insns(-(1 << 25)); - check_bytes("00000096", |cb| bl(cb, offset)); + let cb = compile(|cb| bl(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: bl #0xfffffffff8000000"); + assert_snapshot!(cb.hexdump(), @"00000096"); } #[test] @@ -1292,7 +1415,7 @@ mod tests { fn test_bl_too_big() { // There are 26 bits available let offset = InstructionOffset::from_insns(1 << 25); - check_bytes("", |cb| bl(cb, offset)); + compile(|cb| bl(cb, offset)); } #[test] @@ -1300,380 +1423,559 @@ mod tests { fn test_bl_too_small() { // There are 26 bits available let offset = InstructionOffset::from_insns(-(1 << 25) - 1); - check_bytes("", |cb| bl(cb, offset)); + compile(|cb| bl(cb, offset)); } #[test] fn test_blr() { - check_bytes("80023fd6", |cb| blr(cb, X20)); + let cb = compile(|cb| blr(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: blr x20"); + assert_snapshot!(cb.hexdump(), @"80023fd6"); } #[test] fn test_br() { - check_bytes("80021fd6", |cb| br(cb, X20)); + let cb = compile(|cb| br(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: br x20"); + assert_snapshot!(cb.hexdump(), @"80021fd6"); } #[test] fn test_cbz() { let offset = InstructionOffset::from_insns(-1); - check_bytes("e0ffffb4e0ffff34", |cb| { + let cb = compile(|cb| { cbz(cb, X0, offset); cbz(cb, W0, offset); }); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: cbz x0, #0xfffffffffffffffc + 0x4: cbz w0, #0 + "); + assert_snapshot!(cb.hexdump(), @"e0ffffb4e0ffff34"); } #[test] fn test_cbnz() { let offset = InstructionOffset::from_insns(2); - check_bytes("540000b554000035", |cb| { + let cb = compile(|cb| { cbnz(cb, X20, offset); cbnz(cb, W20, offset); }); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: cbnz x20, #8 + 0x4: cbnz w20, #0xc + "); + assert_snapshot!(cb.hexdump(), @"540000b554000035"); } #[test] fn test_brk_none() { - check_bytes("00003ed4", |cb| brk(cb, A64Opnd::None)); + let cb = compile(|cb| brk(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xf000"); + assert_snapshot!(cb.hexdump(), @"00003ed4"); } #[test] fn test_brk_uimm() { - check_bytes("c00120d4", |cb| brk(cb, A64Opnd::new_uimm(14))); + let cb = compile(|cb| brk(cb, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xe"); + assert_snapshot!(cb.hexdump(), @"c00120d4"); } #[test] fn test_cmp_register() { - check_bytes("5f010beb", |cb| cmp(cb, X10, X11)); + let cb = compile(|cb| cmp(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, x11"); + assert_snapshot!(cb.hexdump(), @"5f010beb"); } #[test] fn test_cmp_immediate() { - check_bytes("5f3900f1", |cb| cmp(cb, X10, A64Opnd::new_uimm(14))); + let cb = compile(|cb| cmp(cb, X10, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, #0xe"); + assert_snapshot!(cb.hexdump(), @"5f3900f1"); } #[test] fn test_csel() { - check_bytes("6a018c9a", |cb| csel(cb, X10, X11, X12, Condition::EQ)); + let cb = compile(|cb| csel(cb, X10, X11, X12, Condition::EQ)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: csel x10, x11, x12, eq"); + assert_snapshot!(cb.hexdump(), @"6a018c9a"); } #[test] fn test_eor_register() { - check_bytes("6a010cca", |cb| eor(cb, X10, X11, X12)); + let cb = compile(|cb| eor(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010cca"); } #[test] fn test_eor_immediate() { - check_bytes("6a0940d2", |cb| eor(cb, X10, X11, A64Opnd::new_uimm(7))); + let cb = compile(|cb| eor(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940d2"); } #[test] fn test_eor_32b_immediate() { - check_bytes("29040152", |cb| eor(cb, W9, W1, A64Opnd::new_uimm(0x80000001))); + let cb = compile(|cb| eor(cb, W9, W1, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor w9, w1, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"29040152"); } #[test] fn test_ldaddal() { - check_bytes("8b01eaf8", |cb| ldaddal(cb, X10, X11, X12)); + let cb = compile(|cb| ldaddal(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaddal x10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8b01eaf8"); } #[test] fn test_ldaxr() { - check_bytes("6afd5fc8", |cb| ldaxr(cb, X10, X11)); + let cb = compile(|cb| ldaxr(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaxr x10, [x11]"); + assert_snapshot!(cb.hexdump(), @"6afd5fc8"); } #[test] fn test_ldp() { - check_bytes("8a2d4da9", |cb| ldp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| ldp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d4da9"); } #[test] fn test_ldp_pre() { - check_bytes("8a2dcda9", |cb| ldp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| ldp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2dcda9"); } #[test] fn test_ldp_post() { - check_bytes("8a2dcda8", |cb| ldp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| ldp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2dcda8"); } #[test] fn test_ldr() { - check_bytes("6a696cf8", |cb| ldr(cb, X10, X11, X12)); + let cb = compile(|cb| ldr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, x12]"); + assert_snapshot!(cb.hexdump(), @"6a696cf8"); } #[test] fn test_ldr_literal() { - check_bytes("40010058", |cb| ldr_literal(cb, X0, 10.into())); + let cb = compile(|cb| ldr_literal(cb, X0, 10.into())); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x0, #0x28"); + assert_snapshot!(cb.hexdump(), @"40010058"); } #[test] fn test_ldr_post() { - check_bytes("6a0541f8", |cb| ldr_post(cb, X10, A64Opnd::new_mem(64, X11, 16))); + let cb = compile(|cb| ldr_post(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11], #0x10"); + assert_snapshot!(cb.hexdump(), @"6a0541f8"); } #[test] fn test_ldr_pre() { - check_bytes("6a0d41f8", |cb| ldr_pre(cb, X10, A64Opnd::new_mem(64, X11, 16))); + let cb = compile(|cb| ldr_pre(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, #0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d41f8"); } #[test] fn test_ldrh() { - check_bytes("6a194079", |cb| ldrh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| ldrh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a194079"); } #[test] fn test_ldrh_pre() { - check_bytes("6acd4078", |cb| ldrh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| ldrh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd4078"); } #[test] fn test_ldrh_post() { - check_bytes("6ac54078", |cb| ldrh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| ldrh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac54078"); } #[test] fn test_ldurh_memory() { - check_bytes("2a004078", |cb| ldurh(cb, W10, A64Opnd::new_mem(64, X1, 0))); - check_bytes("2ab04778", |cb| ldurh(cb, W10, A64Opnd::new_mem(64, X1, 123))); + let cb = compile(|cb| { + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 0)); + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 123)); + }); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldurh w10, [x1] + 0x4: ldurh w10, [x1, #0x7b] + "); + assert_snapshot!(cb.hexdump(), @"2a0040782ab04778"); } #[test] fn test_ldur_memory() { - check_bytes("20b047f8", |cb| ldur(cb, X0, A64Opnd::new_mem(64, X1, 123))); + let cb = compile(|cb| ldur(cb, X0, A64Opnd::new_mem(64, X1, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"20b047f8"); } #[test] fn test_ldur_register() { - check_bytes("200040f8", |cb| ldur(cb, X0, X1)); + let cb = compile(|cb| ldur(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1]"); + assert_snapshot!(cb.hexdump(), @"200040f8"); } #[test] fn test_ldursw() { - check_bytes("6ab187b8", |cb| ldursw(cb, X10, A64Opnd::new_mem(64, X11, 123))); + let cb = compile(|cb| ldursw(cb, X10, A64Opnd::new_mem(64, X11, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldursw x10, [x11, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"6ab187b8"); } #[test] fn test_lsl() { - check_bytes("6ac572d3", |cb| lsl(cb, X10, X11, A64Opnd::new_uimm(14))); + let cb = compile(|cb| lsl(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsl x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6ac572d3"); } #[test] fn test_lsr() { - check_bytes("6afd4ed3", |cb| lsr(cb, X10, X11, A64Opnd::new_uimm(14))); + let cb = compile(|cb| lsr(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsr x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6afd4ed3"); } #[test] fn test_mov_registers() { - check_bytes("ea030baa", |cb| mov(cb, X10, X11)); + let cb = compile(|cb| mov(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea030baa"); } #[test] fn test_mov_immediate() { - check_bytes("eaf300b2", |cb| mov(cb, X10, A64Opnd::new_uimm(0x5555555555555555))); + let cb = compile(|cb| mov(cb, X10, A64Opnd::new_uimm(0x5555555555555555))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, xzr, #0x5555555555555555"); + assert_snapshot!(cb.hexdump(), @"eaf300b2"); } #[test] fn test_mov_32b_immediate() { - check_bytes("ea070132", |cb| mov(cb, W10, A64Opnd::new_uimm(0x80000001))); + let cb = compile(|cb| mov(cb, W10, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov w10, #-0x7fffffff"); + assert_snapshot!(cb.hexdump(), @"ea070132"); } #[test] fn test_mov_into_sp() { - check_bytes("1f000091", |cb| mov(cb, X31, X0)); + let cb = compile(|cb| mov(cb, X31, X0)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov sp, x0"); + assert_snapshot!(cb.hexdump(), @"1f000091"); } #[test] fn test_mov_from_sp() { - check_bytes("e0030091", |cb| mov(cb, X0, X31)); + let cb = compile(|cb| mov(cb, X0, X31)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, sp"); + assert_snapshot!(cb.hexdump(), @"e0030091"); } #[test] fn test_movk() { - check_bytes("600fa0f2", |cb| movk(cb, X0, A64Opnd::new_uimm(123), 16)); + let cb = compile(|cb| movk(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: movk x0, #0x7b, lsl #16"); + assert_snapshot!(cb.hexdump(), @"600fa0f2"); + } + + #[test] + fn test_movn() { + let cb = compile(|cb| movn(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #-0x7b0001"); + assert_snapshot!(cb.hexdump(), @"600fa092"); } #[test] fn test_movz() { - check_bytes("600fa0d2", |cb| movz(cb, X0, A64Opnd::new_uimm(123), 16)); + let cb = compile(|cb| movz(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #0x7b0000"); + assert_snapshot!(cb.hexdump(), @"600fa0d2"); } #[test] fn test_mrs() { - check_bytes("0a423bd5", |cb| mrs(cb, X10, SystemRegister::NZCV)); + let cb = compile(|cb| mrs(cb, X10, SystemRegister::NZCV)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mrs x10, nzcv"); + assert_snapshot!(cb.hexdump(), @"0a423bd5"); } #[test] fn test_msr() { - check_bytes("0a421bd5", |cb| msr(cb, SystemRegister::NZCV, X10)); + let cb = compile(|cb| msr(cb, SystemRegister::NZCV, X10)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: msr nzcv, x10"); + assert_snapshot!(cb.hexdump(), @"0a421bd5"); } #[test] fn test_mul() { - check_bytes("6a7d0c9b", |cb| mul(cb, X10, X11, X12)); + let cb = compile(|cb| mul(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mul x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a7d0c9b"); } #[test] fn test_mvn() { - check_bytes("ea032baa", |cb| mvn(cb, X10, X11)); + let cb = compile(|cb| mvn(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mvn x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea032baa"); } #[test] fn test_nop() { - check_bytes("1f2003d5", |cb| nop(cb)); + let cb = compile(nop); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: nop"); + assert_snapshot!(cb.hexdump(), @"1f2003d5"); } #[test] fn test_orn() { - check_bytes("6a012caa", |cb| orn(cb, X10, X11, X12)); + let cb = compile(|cb| orn(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orn x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a012caa"); } #[test] fn test_orr_register() { - check_bytes("6a010caa", |cb| orr(cb, X10, X11, X12)); + let cb = compile(|cb| orr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010caa"); } #[test] fn test_orr_immediate() { - check_bytes("6a0940b2", |cb| orr(cb, X10, X11, A64Opnd::new_uimm(7))); + let cb = compile(|cb| orr(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940b2"); } #[test] fn test_orr_32b_immediate() { - check_bytes("6a010032", |cb| orr(cb, W10, W11, A64Opnd::new_uimm(1))); + let cb = compile(|cb| orr(cb, W10, W11, A64Opnd::new_uimm(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr w10, w11, #1"); + assert_snapshot!(cb.hexdump(), @"6a010032"); } #[test] fn test_ret_none() { - check_bytes("c0035fd6", |cb| ret(cb, A64Opnd::None)); + let cb = compile(|cb| ret(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c0035fd6"); } #[test] fn test_ret_register() { - check_bytes("80025fd6", |cb| ret(cb, X20)); + let cb = compile(|cb| ret(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret x20"); + assert_snapshot!(cb.hexdump(), @"80025fd6"); } #[test] fn test_stlxr() { - check_bytes("8bfd0ac8", |cb| stlxr(cb, W10, X11, X12)); + let cb = compile(|cb| stlxr(cb, W10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stlxr w10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8bfd0ac8"); } #[test] fn test_stp() { - check_bytes("8a2d0da9", |cb| stp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| stp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d0da9"); } #[test] fn test_stp_pre() { - check_bytes("8a2d8da9", |cb| stp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| stp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2d8da9"); } #[test] fn test_stp_post() { - check_bytes("8a2d8da8", |cb| stp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| stp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2d8da8"); } #[test] fn test_str_post() { - check_bytes("6a051ff8", |cb| str_post(cb, X10, A64Opnd::new_mem(64, X11, -16))); + let cb = compile(|cb| str_post(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11], #0xfffffffffffffff0"); + assert_snapshot!(cb.hexdump(), @"6a051ff8"); } #[test] fn test_str_pre() { - check_bytes("6a0d1ff8", |cb| str_pre(cb, X10, A64Opnd::new_mem(64, X11, -16))); + let cb = compile(|cb| str_pre(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11, #-0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d1ff8"); } #[test] fn test_strh() { - check_bytes("6a190079", |cb| strh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| strh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a190079"); } #[test] fn test_strh_pre() { - check_bytes("6acd0078", |cb| strh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| strh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd0078"); } #[test] fn test_strh_post() { - check_bytes("6ac50078", |cb| strh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| strh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac50078"); } #[test] fn test_stur_64_bits() { - check_bytes("6a0108f8", |cb| stur(cb, X10, A64Opnd::new_mem(64, X11, 128))); + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(64, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur x10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108f8"); } #[test] fn test_stur_32_bits() { - check_bytes("6a0108b8", |cb| stur(cb, X10, A64Opnd::new_mem(32, X11, 128))); + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(32, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur w10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108b8"); } #[test] fn test_sub_reg() { - check_bytes("200002cb", |cb| sub(cb, X0, X1, X2)); + let cb = compile(|cb| sub(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002cb"); } #[test] fn test_sub_uimm() { - check_bytes("201c00d1", |cb| sub(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); } #[test] fn test_sub_imm_positive() { - check_bytes("201c00d1", |cb| sub(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); } #[test] fn test_sub_imm_negative() { - check_bytes("201c0091", |cb| sub(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); } #[test] fn test_subs_reg() { - check_bytes("200002eb", |cb| subs(cb, X0, X1, X2)); + let cb = compile(|cb| subs(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002eb"); } #[test] fn test_subs_imm_positive() { - check_bytes("201c00f1", |cb| subs(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); } #[test] fn test_subs_imm_negative() { - check_bytes("201c00b1", |cb| subs(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); } #[test] fn test_subs_uimm() { - check_bytes("201c00f1", |cb| subs(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); } #[test] fn test_sxtw() { - check_bytes("6a7d4093", |cb| sxtw(cb, X10, W11)); + let cb = compile(|cb| sxtw(cb, X10, W11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sxtw x10, w11"); + assert_snapshot!(cb.hexdump(), @"6a7d4093"); } #[test] fn test_tbnz() { - check_bytes("4a005037", |cb| tbnz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + let cb = compile(|cb| tbnz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbnz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005037"); } #[test] fn test_tbz() { - check_bytes("4a005036", |cb| tbz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + let cb = compile(|cb| tbz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005036"); } #[test] fn test_tst_register() { - check_bytes("1f0001ea", |cb| tst(cb, X0, X1)); + let cb = compile(|cb| tst(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); } #[test] fn test_tst_immediate() { - check_bytes("3f0840f2", |cb| tst(cb, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| tst(cb, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x1, #7"); + assert_snapshot!(cb.hexdump(), @"3f0840f2"); } #[test] fn test_tst_32b_immediate() { - check_bytes("1f3c0072", |cb| tst(cb, W0, A64Opnd::new_uimm(0xffff))); + let cb = compile(|cb| tst(cb, W0, A64Opnd::new_uimm(0xffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0xffff"); + assert_snapshot!(cb.hexdump(), @"1f3c0072"); + } + + #[test] + fn test_add_extend_various_regs() { + let mut cb = CodeBlock::new_dummy(); + + add_extended(&mut cb, X10, X11, X9); + add_extended(&mut cb, X30, X30, X30); + add_extended(&mut cb, X31, X31, X31); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x10, x11, x9, uxtx + 0x4: add x30, x30, x30, uxtx + 0x8: add sp, sp, xzr + "); + assert_snapshot!(cb.hexdump(), @"6a61298bde633e8bff633f8b"); } } -*/ diff --git a/zjit/src/asm/arm64/opnd.rs b/zjit/src/asm/arm64/opnd.rs index 6e31851504..667533ab93 100644 --- a/zjit/src/asm/arm64/opnd.rs +++ b/zjit/src/asm/arm64/opnd.rs @@ -1,4 +1,4 @@ - +use std::fmt; /// This operand represents a register. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -18,7 +18,7 @@ impl A64Reg { } } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct A64Mem { // Size in bits @@ -42,7 +42,7 @@ impl A64Mem { } } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum A64Opnd { // Dummy operand @@ -79,10 +79,7 @@ impl A64Opnd { /// Convenience function to check if this operand is a register. pub fn is_reg(&self) -> bool { - match self { - A64Opnd::Reg(_) => true, - _ => false - } + matches!(self, A64Opnd::Reg(_)) } /// Unwrap a register from an operand. @@ -119,6 +116,12 @@ pub const X20_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 20 }; pub const X21_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 21 }; pub const X22_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 22 }; +// frame pointer (base pointer) +pub const X29_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 29 }; + +// link register +pub const X30_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 30 }; + // zero register pub const XZR_REG: A64Reg = A64Reg { num_bits: 64, reg_no: 31 }; @@ -153,7 +156,7 @@ pub const X26: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 26 }); pub const X27: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 27 }); pub const X28: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 28 }); pub const X29: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 29 }); -pub const X30: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: 30 }); +pub const X30: A64Opnd = A64Opnd::Reg(X30_REG); pub const X31: A64Opnd = A64Opnd::Reg(XZR_REG); // 32-bit registers @@ -193,3 +196,75 @@ pub const W31: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 31 }); // C argument registers pub const C_ARG_REGS: [A64Opnd; 4] = [X0, X1, X2, X3]; pub const C_ARG_REGREGS: [A64Reg; 4] = [X0_REG, X1_REG, X2_REG, X3_REG]; + +impl fmt::Display for A64Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match A64Opnd::Reg(*self) { + X0 => write!(f, "x0"), + X1 => write!(f, "x1"), + X2 => write!(f, "x2"), + X3 => write!(f, "x3"), + X4 => write!(f, "x4"), + X5 => write!(f, "x5"), + X6 => write!(f, "x6"), + X7 => write!(f, "x7"), + X8 => write!(f, "x8"), + X9 => write!(f, "x9"), + X10 => write!(f, "x10"), + X11 => write!(f, "x11"), + X12 => write!(f, "x12"), + X13 => write!(f, "x13"), + X14 => write!(f, "x14"), + X15 => write!(f, "x15"), + X16 => write!(f, "x16"), + X17 => write!(f, "x17"), + X18 => write!(f, "x18"), + X19 => write!(f, "x19"), + X20 => write!(f, "x20"), + X21 => write!(f, "x21"), + X22 => write!(f, "x22"), + X23 => write!(f, "x23"), + X24 => write!(f, "x24"), + X25 => write!(f, "x25"), + X26 => write!(f, "x26"), + X27 => write!(f, "x27"), + X28 => write!(f, "x28"), + X29 => write!(f, "x29"), + X30 => write!(f, "x30"), + X31 => write!(f, "x31"), + W0 => write!(f, "w0"), + W1 => write!(f, "w1"), + W2 => write!(f, "w2"), + W3 => write!(f, "w3"), + W4 => write!(f, "w4"), + W5 => write!(f, "w5"), + W6 => write!(f, "w6"), + W7 => write!(f, "w7"), + W8 => write!(f, "w8"), + W9 => write!(f, "w9"), + W10 => write!(f, "w10"), + W11 => write!(f, "w11"), + W12 => write!(f, "w12"), + W13 => write!(f, "w13"), + W14 => write!(f, "w14"), + W15 => write!(f, "w15"), + W16 => write!(f, "w16"), + W17 => write!(f, "w17"), + W18 => write!(f, "w18"), + W19 => write!(f, "w19"), + W20 => write!(f, "w20"), + W21 => write!(f, "w21"), + W22 => write!(f, "w22"), + W23 => write!(f, "w23"), + W24 => write!(f, "w24"), + W25 => write!(f, "w25"), + W26 => write!(f, "w26"), + W27 => write!(f, "w27"), + W28 => write!(f, "w28"), + W29 => write!(f, "w29"), + W30 => write!(f, "w30"), + W31 => write!(f, "w31"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs index a7f2705af1..9049624529 100644 --- a/zjit/src/asm/mod.rs +++ b/zjit/src/asm/mod.rs @@ -1,5 +1,8 @@ +//! Model for creating generating textual assembler code. + use std::collections::BTreeMap; -//use std::fmt; +use std::fmt; +use std::ops::Range; use std::rc::Rc; use std::cell::RefCell; use std::mem; @@ -7,15 +10,19 @@ use crate::virtualmem::*; // Lots of manual vertical alignment in there that rustfmt doesn't handle well. #[rustfmt::skip] +#[cfg(target_arch = "x86_64")] pub mod x86_64; +#[cfg(target_arch = "aarch64")] pub mod arm64; /// Index to a label created by cb.new_label() -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] pub struct Label(pub usize); +/// The object that knows how to encode the branch instruction. +type BranchEncoder = Box<dyn Fn(&mut CodeBlock, i64, i64)>; + /// Reference to an ASM label -#[derive(Clone)] pub struct LabelRef { // Position in the code block where the label reference exists pos: usize, @@ -29,7 +36,7 @@ pub struct LabelRef { num_bytes: usize, /// The object that knows how to encode the branch instruction. - encode: fn(&mut CodeBlock, i64, i64) + encode: BranchEncoder, } /// Block of memory into which instructions can be assembled @@ -81,6 +88,11 @@ impl CodeBlock { } } + /// Size of the region in bytes that we have allocated physical memory for. + pub fn mapped_region_size(&self) -> usize { + self.mem_block.borrow().mapped_region_size() + } + /// Add an assembly comment if the feature is on. pub fn add_comment(&mut self, comment: &str) { if !self.keep_comments { @@ -106,19 +118,23 @@ impl CodeBlock { self.write_pos } + pub fn write_mem(&self, write_ptr: CodePtr, byte: u8) -> Result<(), WriteError> { + self.mem_block.borrow_mut().write_byte(write_ptr, byte) + } + /// Get a (possibly dangling) direct pointer to the current write position pub fn get_write_ptr(&self) -> CodePtr { self.get_ptr(self.write_pos) } /// Set the current write position from a pointer - fn set_write_ptr(&mut self, code_ptr: CodePtr) { + pub fn set_write_ptr(&mut self, code_ptr: CodePtr) { let pos = code_ptr.as_offset() - self.mem_block.borrow().start_ptr().as_offset(); self.write_pos = pos.try_into().unwrap(); } /// Invoke a callback with write_ptr temporarily adjusted to a given address - pub fn with_write_ptr(&mut self, code_ptr: CodePtr, callback: impl Fn(&mut CodeBlock)) { + pub fn with_write_ptr(&mut self, code_ptr: CodePtr, callback: impl Fn(&mut CodeBlock)) -> Range<CodePtr> { // Temporarily update the write_pos. Ignore the dropped_bytes flag at the old address. let old_write_pos = self.write_pos; let old_dropped_bytes = self.dropped_bytes; @@ -128,9 +144,13 @@ impl CodeBlock { // Invoke the callback callback(self); + // Build a code range modified by the callback + let ret = code_ptr..self.get_write_ptr(); + // Restore the original write_pos and dropped_bytes flag. self.dropped_bytes = old_dropped_bytes; self.write_pos = old_write_pos; + ret } /// Get a (possibly dangling) direct pointer into the executable memory block @@ -188,6 +208,14 @@ impl CodeBlock { self.dropped_bytes } + /// Set dropped_bytes to false if the current zjit_alloc_bytes() + code_region_size + /// + page_size is below --zjit-mem-size. + pub fn update_dropped_bytes(&mut self) { + if self.mem_block.borrow().can_allocate() { + self.dropped_bytes = false; + } + } + /// Allocate a new label with a given name pub fn new_label(&mut self, name: String) -> Label { assert!(!name.contains(' '), "use underscores in label names, not spaces"); @@ -205,11 +233,11 @@ impl CodeBlock { } // Add a label reference at the current write position - pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: fn(&mut CodeBlock, i64, i64)) { + pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: impl Fn(&mut CodeBlock, i64, i64) + 'static) { assert!(label.0 < self.label_addrs.len()); // Keep track of the reference - self.label_refs.push(LabelRef { pos: self.write_pos, label, num_bytes, encode }); + self.label_refs.push(LabelRef { pos: self.write_pos, label, num_bytes, encode: Box::new(encode) }); // Move past however many bytes the instruction takes up if self.write_pos + num_bytes < self.mem_size { @@ -233,7 +261,7 @@ impl CodeBlock { assert!(label_addr < self.mem_size); self.write_pos = ref_pos; - (label_ref.encode)(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); + (label_ref.encode.as_ref())(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); // Assert that we've written the same number of bytes that we // expected to have written. @@ -248,6 +276,11 @@ impl CodeBlock { assert!(self.label_refs.is_empty()); } + /// Convert a Label to CodePtr + pub fn resolve_label(&self, label: Label) -> CodePtr { + self.get_ptr(self.label_addrs[label.0]) + } + pub fn clear_labels(&mut self) { self.label_addrs.clear(); self.label_names.clear(); @@ -258,21 +291,86 @@ impl CodeBlock { pub fn mark_all_executable(&mut self) { self.mem_block.borrow_mut().mark_all_executable(); } + + /// Call a func with the disasm of generated code for testing + #[allow(unused_variables)] + #[cfg(all(test, feature = "disasm"))] + pub fn disasm(&self) -> String { + let start_addr = self.get_ptr(0).raw_addr(self); + let end_addr = self.get_write_ptr().raw_addr(self); + crate::disasm::disasm_addr_range(self, start_addr, end_addr) + } + + /// Return the hex dump of generated code for testing + #[cfg(test)] + pub fn hexdump(&self) -> String { + format!("{:x}", self) + } +} + +/// Run assert_snapshot! only if cfg!(feature = "disasm"). +/// $actual can be not only `cb.disasm()` but also `disasms!(cb1, cb2, ...)`. +#[cfg(test)] +#[macro_export] +macro_rules! assert_disasm_snapshot { + ($actual: expr, @$($tt: tt)*) => {{ + #[cfg(feature = "disasm")] + assert_snapshot!($actual, @$($tt)*) + }}; +} + +/// Combine multiple cb.disasm() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms { + ($( $cb:expr ),+ $(,)?) => {{ + crate::disasms_with!("", $( $cb ),+) + }}; +} + +/// Basically `disasms!` but allows a non-"" delimiter, such as "\n" +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms_with { + ($join:expr, $( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.disasm() ),+].join($join) + }}; +} + +/// Combine multiple cb.hexdump() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(test)] +#[macro_export] +macro_rules! hexdumps { + ($( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.hexdump() ),+].join("\n") + }}; +} + +/// Produce hex string output from the bytes in a code block +impl fmt::LowerHex for CodeBlock { + fn fmt(&self, fmtr: &mut fmt::Formatter) -> fmt::Result { + for pos in 0..self.write_pos { + let mem_block = &*self.mem_block.borrow(); + let byte = unsafe { mem_block.start_ptr().raw_ptr(mem_block).add(pos).read() }; + fmtr.write_fmt(format_args!("{byte:02x}"))?; + } + Ok(()) + } } #[cfg(test)] impl CodeBlock { /// Stubbed CodeBlock for testing. Can't execute generated code. pub fn new_dummy() -> Self { - use std::ptr::NonNull; - use crate::virtualmem::*; - use crate::virtualmem::tests::TestingAllocator; - - let mem_size = 1024; - let alloc = TestingAllocator::new(mem_size); - let mem_start: *const u8 = alloc.mem_start(); - let virt_mem = VirtualMem::new(alloc, 1, NonNull::new(mem_start as *mut u8).unwrap(), mem_size, 128 * 1024 * 1024); + const DEFAULT_MEM_SIZE: usize = 1024 * 1024; + CodeBlock::new_dummy_sized(DEFAULT_MEM_SIZE) + } + pub fn new_dummy_sized(mem_size: usize) -> Self { + use crate::virtualmem::*; + let virt_mem = VirtualMem::alloc(mem_size, None); Self::new(Rc::new(RefCell::new(virt_mem)), false) } } @@ -297,7 +395,7 @@ pub fn imm_num_bits(imm: i64) -> u8 return 32; } - return 64; + 64 } /// Compute the number of bits needed to encode an unsigned value @@ -314,10 +412,9 @@ pub fn uimm_num_bits(uimm: u64) -> u8 return 32; } - return 64; + 64 } -/* #[cfg(test)] mod tests { @@ -353,32 +450,4 @@ mod tests assert_eq!(uimm_num_bits((u32::MAX as u64) + 1), 64); assert_eq!(uimm_num_bits(u64::MAX), 64); } - - #[test] - fn test_code_size() { - // Write 4 bytes in the first page - let mut cb = CodeBlock::new_dummy(CodeBlock::PREFERRED_CODE_PAGE_SIZE * 2); - cb.write_bytes(&[0, 0, 0, 0]); - assert_eq!(cb.code_size(), 4); - - // Moving to the next page should not increase code_size - cb.next_page(cb.get_write_ptr(), |_, _| {}); - assert_eq!(cb.code_size(), 4); - - // Write 4 bytes in the second page - cb.write_bytes(&[0, 0, 0, 0]); - assert_eq!(cb.code_size(), 8); - - // Rewrite 4 bytes in the first page - let old_write_pos = cb.get_write_pos(); - cb.set_pos(0); - cb.write_bytes(&[1, 1, 1, 1]); - - // Moving from an old page to the next page should not increase code_size - cb.next_page(cb.get_write_ptr(), |_, _| {}); - cb.set_pos(old_write_pos); - assert_eq!(cb.code_size(), 8); - } } - -*/ diff --git a/zjit/src/asm/x86_64/mod.rs b/zjit/src/asm/x86_64/mod.rs index efc58dfdb8..cfedca4540 100644 --- a/zjit/src/asm/x86_64/mod.rs +++ b/zjit/src/asm/x86_64/mod.rs @@ -47,7 +47,7 @@ pub struct X86Reg pub reg_no: u8, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct X86Mem { // Size in bits @@ -66,7 +66,7 @@ pub struct X86Mem pub disp: i32, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum X86Opnd { // Dummy operand @@ -163,10 +163,7 @@ impl X86Opnd { } pub fn is_some(&self) -> bool { - match self { - X86Opnd::None => false, - _ => true - } + !matches!(self, X86Opnd::None) } } @@ -284,11 +281,11 @@ pub fn mem_opnd(num_bits: u8, base_reg: X86Opnd, disp: i32) -> X86Opnd } else { X86Opnd::Mem( X86Mem { - num_bits: num_bits, + num_bits, base_reg_no: base_reg.reg_no, idx_reg_no: None, scale_exp: 0, - disp: disp, + disp, } ) } @@ -317,34 +314,6 @@ pub fn mem_opnd_sib(num_bits: u8, base_opnd: X86Opnd, index_opnd: X86Opnd, scale } } -/* -// Struct member operand -#define member_opnd(base_reg, struct_type, member_name) mem_opnd( \ - 8 * sizeof(((struct_type*)0)->member_name), \ - base_reg, \ - offsetof(struct_type, member_name) \ -) - -// Struct member operand with an array index -#define member_opnd_idx(base_reg, struct_type, member_name, idx) mem_opnd( \ - 8 * sizeof(((struct_type*)0)->member_name[0]), \ - base_reg, \ - (offsetof(struct_type, member_name) + \ - sizeof(((struct_type*)0)->member_name[0]) * idx) \ -) -*/ - -/* -// TODO: this should be a method, X86Opnd.resize() or X86Opnd.subreg() -static x86opnd_t resize_opnd(x86opnd_t opnd, uint32_t num_bits) -{ - assert (num_bits % 8 == 0); - x86opnd_t sub = opnd; - sub.num_bits = num_bits; - return sub; -} -*/ - pub fn imm_opnd(value: i64) -> X86Opnd { X86Opnd::Imm(X86Imm { num_bits: imm_num_bits(value), value }) @@ -540,7 +509,7 @@ fn write_rm_unary(cb: &mut CodeBlock, op_mem_reg_8: u8, op_mem_reg_pref: u8, op_ // Encode an add-like RM instruction with multiple possible encodings fn write_rm_multi(cb: &mut CodeBlock, op_mem_reg8: u8, op_mem_reg_pref: u8, op_reg_mem8: u8, op_reg_mem_pref: u8, op_mem_imm8: u8, op_mem_imm_sml: u8, op_mem_imm_lrg: u8, op_ext_imm: Option<u8>, opnd0: X86Opnd, opnd1: X86Opnd) { - assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_)), "unexpected opnd0: {opnd0:?}, {opnd1:?}"); // Check the size of opnd0 let opnd_size = opnd0.num_bits(); @@ -810,13 +779,6 @@ pub fn imul(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { write_rm(cb, false, true, opnd0, opnd1, None, &[0x0F, 0xAF]); } - // Flip the operands to handle this case. This instruction has weird encoding restrictions. - (X86Opnd::Mem(_), X86Opnd::Reg(_)) => { - //REX.W + 0F AF /rIMUL r64, r/m64 - // Quadword register := Quadword register * r/m64. - write_rm(cb, false, true, opnd1, opnd0, None, &[0x0F, 0xAF]); - } - _ => unreachable!() } } @@ -1024,7 +986,10 @@ pub fn mov(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { } let output_num_bits:u32 = if mem.num_bits > 32 { 32 } else { mem.num_bits.into() }; - assert!(imm_num_bits(imm.value) <= (output_num_bits as u8)); + assert!( + mem.num_bits < 64 || imm_num_bits(imm.value) <= (output_num_bits as u8), + "immediate value should be small enough to survive sign extension" + ); cb.write_int(imm.value as u64, output_num_bits); }, // M + UImm @@ -1039,7 +1004,10 @@ pub fn mov(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { } let output_num_bits = if mem.num_bits > 32 { 32 } else { mem.num_bits.into() }; - assert!(imm_num_bits(uimm.value as i64) <= (output_num_bits as u8)); + assert!( + mem.num_bits < 64 || imm_num_bits(uimm.value as i64) <= (output_num_bits as u8), + "immediate value should be small enough to survive sign extension" + ); cb.write_int(uimm.value, output_num_bits); }, // * + Imm/UImm @@ -1097,46 +1065,6 @@ pub fn movsx(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { } } -/* -/// movzx - Move with zero extension (unsigned values) -void movzx(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) -{ - cb.writeASM("movzx", dst, src); - - uint32_t dstSize; - if (dst.isReg) - dstSize = dst.reg.size; - else - assert (false, "movzx dst must be a register"); - - uint32_t srcSize; - if (src.isReg) - srcSize = src.reg.size; - else if (src.isMem) - srcSize = src.mem.size; - else - assert (false); - - assert ( - srcSize < dstSize, - "movzx: srcSize >= dstSize" - ); - - if (srcSize is 8) - { - cb.writeRMInstr!('r', 0xFF, 0x0F, 0xB6)(dstSize is 16, dstSize is 64, dst, src); - } - else if (srcSize is 16) - { - cb.writeRMInstr!('r', 0xFF, 0x0F, 0xB7)(dstSize is 16, dstSize is 64, dst, src); - } - else - { - assert (false, "invalid src operand size for movxz"); - } -} -*/ - /// nop - Noop, one or multiple bytes long pub fn nop(cb: &mut CodeBlock, length: u32) { match length { @@ -1399,7 +1327,7 @@ pub fn test(cb: &mut CodeBlock, rm_opnd: X86Opnd, test_opnd: X86Opnd) { write_rm(cb, rm_num_bits == 16, rm_num_bits == 64, test_opnd, rm_opnd, None, &[0x85]); } }, - _ => unreachable!() + _ => unreachable!("unexpected operands for test: {rm_opnd:?}, {test_opnd:?}") }; } @@ -1445,3 +1373,74 @@ pub fn xor(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { opnd1 ); } + +impl fmt::Display for X86Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match X86Opnd::Reg(*self) { + RAX => write!(f, "rax"), + RCX => write!(f, "rcx"), + RDX => write!(f, "rdx"), + RBX => write!(f, "rbx"), + RSP => write!(f, "rsp"), + RBP => write!(f, "rbp"), + RSI => write!(f, "rsi"), + RDI => write!(f, "rdi"), + R8 => write!(f, "r8"), + R9 => write!(f, "r9"), + R10 => write!(f, "r10"), + R11 => write!(f, "r11"), + R12 => write!(f, "r12"), + R13 => write!(f, "r13"), + R14 => write!(f, "r14"), + R15 => write!(f, "r15"), + EAX => write!(f, "eax"), + ECX => write!(f, "ecx"), + EDX => write!(f, "edx"), + EBX => write!(f, "ebx"), + ESP => write!(f, "esp"), + EBP => write!(f, "ebp"), + ESI => write!(f, "esi"), + EDI => write!(f, "edi"), + R8D => write!(f, "r8d"), + R9D => write!(f, "r9d"), + R10D => write!(f, "r10d"), + R11D => write!(f, "r11d"), + R12D => write!(f, "r12d"), + R13D => write!(f, "r13d"), + R14D => write!(f, "r14d"), + R15D => write!(f, "r15d"), + AX => write!(f, "ax"), + CX => write!(f, "cx"), + DX => write!(f, "dx"), + BX => write!(f, "bx"), + BP => write!(f, "bp"), + SI => write!(f, "si"), + DI => write!(f, "di"), + R8W => write!(f, "r8w"), + R9W => write!(f, "r9w"), + R10W => write!(f, "r10w"), + R11W => write!(f, "r11w"), + R12W => write!(f, "r12w"), + R13W => write!(f, "r13w"), + R14W => write!(f, "r14w"), + R15W => write!(f, "r15w"), + AL => write!(f, "al"), + CL => write!(f, "cl"), + DL => write!(f, "dl"), + BL => write!(f, "bl"), + SPL => write!(f, "spl"), + BPL => write!(f, "bpl"), + SIL => write!(f, "sil"), + DIL => write!(f, "dil"), + R8B => write!(f, "r8b"), + R9B => write!(f, "r9b"), + R10B => write!(f, "r10b"), + R11B => write!(f, "r11b"), + R12B => write!(f, "r12b"), + R13B => write!(f, "r13b"), + R14B => write!(f, "r14b"), + R15B => write!(f, "r15b"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/x86_64/tests.rs b/zjit/src/asm/x86_64/tests.rs index f2b949b7f7..d574bdb034 100644 --- a/zjit/src/asm/x86_64/tests.rs +++ b/zjit/src/asm/x86_64/tests.rs @@ -1,372 +1,840 @@ #![cfg(test)] -//use crate::asm::x86_64::*; +use insta::assert_snapshot; + +#[cfg(feature = "disasm")] +use crate::disasms; +use crate::{asm::x86_64::*, hexdumps, assert_disasm_snapshot}; -/* /// Check that the bytes for an instruction sequence match a hex string fn check_bytes<R>(bytes: &str, run: R) where R: FnOnce(&mut super::CodeBlock) { - let mut cb = super::CodeBlock::new_dummy(4096); + let mut cb = super::CodeBlock::new_dummy(); run(&mut cb); assert_eq!(format!("{:x}", cb), bytes); } +fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + cb +} + #[test] fn test_add() { - check_bytes("80c103", |cb| add(cb, CL, imm_opnd(3))); - check_bytes("00d9", |cb| add(cb, CL, BL)); - check_bytes("4000e1", |cb| add(cb, CL, SPL)); - check_bytes("6601d9", |cb| add(cb, CX, BX)); - check_bytes("4801d8", |cb| add(cb, RAX, RBX)); - check_bytes("01d1", |cb| add(cb, ECX, EDX)); - check_bytes("4c01f2", |cb| add(cb, RDX, R14)); - check_bytes("480110", |cb| add(cb, mem_opnd(64, RAX, 0), RDX)); - check_bytes("480310", |cb| add(cb, RDX, mem_opnd(64, RAX, 0))); - check_bytes("48035008", |cb| add(cb, RDX, mem_opnd(64, RAX, 8))); - check_bytes("480390ff000000", |cb| add(cb, RDX, mem_opnd(64, RAX, 255))); - check_bytes("4881407fff000000", |cb| add(cb, mem_opnd(64, RAX, 127), imm_opnd(255))); - check_bytes("0110", |cb| add(cb, mem_opnd(32, RAX, 0), EDX)); - check_bytes("4883c408", |cb| add(cb, RSP, imm_opnd(8))); - check_bytes("83c108", |cb| add(cb, ECX, imm_opnd(8))); - check_bytes("81c1ff000000", |cb| add(cb, ECX, imm_opnd(255))); + let cb01 = compile(|cb| add(cb, CL, imm_opnd(3))); + let cb02 = compile(|cb| add(cb, CL, BL)); + let cb03 = compile(|cb| add(cb, CL, SPL)); + let cb04 = compile(|cb| add(cb, CX, BX)); + let cb05 = compile(|cb| add(cb, RAX, RBX)); + let cb06 = compile(|cb| add(cb, ECX, EDX)); + let cb07 = compile(|cb| add(cb, RDX, R14)); + let cb08 = compile(|cb| add(cb, mem_opnd(64, RAX, 0), RDX)); + let cb09 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 0))); + let cb10 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 8))); + let cb11 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 255))); + let cb12 = compile(|cb| add(cb, mem_opnd(64, RAX, 127), imm_opnd(255))); + let cb13 = compile(|cb| add(cb, mem_opnd(32, RAX, 0), EDX)); + let cb14 = compile(|cb| add(cb, RSP, imm_opnd(8))); + let cb15 = compile(|cb| add(cb, ECX, imm_opnd(8))); + let cb16 = compile(|cb| add(cb, ECX, imm_opnd(255))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @r" + 0x0: add cl, 3 + 0x0: add cl, bl + 0x0: add cl, spl + 0x0: add cx, bx + 0x0: add rax, rbx + 0x0: add ecx, edx + 0x0: add rdx, r14 + 0x0: add qword ptr [rax], rdx + 0x0: add rdx, qword ptr [rax] + 0x0: add rdx, qword ptr [rax + 8] + 0x0: add rdx, qword ptr [rax + 0xff] + 0x0: add qword ptr [rax + 0x7f], 0xff + 0x0: add dword ptr [rax], edx + 0x0: add rsp, 8 + 0x0: add ecx, 8 + 0x0: add ecx, 0xff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @r" + 80c103 + 00d9 + 4000e1 + 6601d9 + 4801d8 + 01d1 + 4c01f2 + 480110 + 480310 + 48035008 + 480390ff000000 + 4881407fff000000 + 0110 + 4883c408 + 83c108 + 81c1ff000000 + "); } #[test] fn test_add_unsigned() { // ADD r/m8, imm8 - check_bytes("4180c001", |cb| add(cb, R8B, uimm_opnd(1))); - check_bytes("4180c07f", |cb| add(cb, R8B, imm_opnd(i8::MAX.try_into().unwrap()))); - + let cb1 = compile(|cb| add(cb, R8B, uimm_opnd(1))); + let cb2 = compile(|cb| add(cb, R8B, imm_opnd(i8::MAX.into()))); // ADD r/m16, imm16 - check_bytes("664183c001", |cb| add(cb, R8W, uimm_opnd(1))); - check_bytes("664181c0ff7f", |cb| add(cb, R8W, uimm_opnd(i16::MAX.try_into().unwrap()))); - + let cb3 = compile(|cb| add(cb, R8W, uimm_opnd(1))); + let cb4 = compile(|cb| add(cb, R8W, uimm_opnd(i16::MAX.try_into().unwrap()))); // ADD r/m32, imm32 - check_bytes("4183c001", |cb| add(cb, R8D, uimm_opnd(1))); - check_bytes("4181c0ffffff7f", |cb| add(cb, R8D, uimm_opnd(i32::MAX.try_into().unwrap()))); - + let cb5 = compile(|cb| add(cb, R8D, uimm_opnd(1))); + let cb6 = compile(|cb| add(cb, R8D, uimm_opnd(i32::MAX.try_into().unwrap()))); // ADD r/m64, imm32 - check_bytes("4983c001", |cb| add(cb, R8, uimm_opnd(1))); - check_bytes("4981c0ffffff7f", |cb| add(cb, R8, uimm_opnd(i32::MAX.try_into().unwrap()))); + let cb7 = compile(|cb| add(cb, R8, uimm_opnd(1))); + let cb8 = compile(|cb| add(cb, R8, uimm_opnd(i32::MAX.try_into().unwrap()))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 0x0: add r8b, 1 + 0x0: add r8b, 0x7f + 0x0: add r8w, 1 + 0x0: add r8w, 0x7fff + 0x0: add r8d, 1 + 0x0: add r8d, 0x7fffffff + 0x0: add r8, 1 + 0x0: add r8, 0x7fffffff + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 4180c001 + 4180c07f + 664183c001 + 664181c0ff7f + 4183c001 + 4181c0ffffff7f + 4983c001 + 4981c0ffffff7f + "); } #[test] fn test_and() { - check_bytes("4421e5", |cb| and(cb, EBP, R12D)); - check_bytes("48832008", |cb| and(cb, mem_opnd(64, RAX, 0), imm_opnd(0x08))); + let cb1 = compile(|cb| and(cb, EBP, R12D)); + let cb2 = compile(|cb| and(cb, mem_opnd(64, RAX, 0), imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: and ebp, r12d + 0x0: and qword ptr [rax], 8 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 4421e5 + 48832008 + "); } #[test] fn test_call_label() { - check_bytes("e8fbffffff", |cb| { + let cb = compile(|cb| { let label_idx = cb.new_label("fn".to_owned()); call_label(cb, label_idx); cb.link_labels(); }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); } #[test] fn test_call_ptr() { // calling a lower address - check_bytes("e8fbffffff", |cb| { + let cb = compile(|cb| { let ptr = cb.get_write_ptr(); call_ptr(cb, RAX, ptr.raw_ptr(cb)); }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); } #[test] fn test_call_reg() { - check_bytes("ffd0", |cb| call(cb, RAX)); + let cb = compile(|cb| call(cb, RAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call rax"); + assert_snapshot!(cb.hexdump(), @"ffd0"); } #[test] fn test_call_mem() { - check_bytes("ff542408", |cb| call(cb, mem_opnd(64, RSP, 8))); + let cb = compile(|cb| call(cb, mem_opnd(64, RSP, 8))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call qword ptr [rsp + 8]"); + assert_snapshot!(cb.hexdump(), @"ff542408"); } #[test] fn test_cmovcc() { - check_bytes("0f4ff7", |cb| cmovg(cb, ESI, EDI)); - check_bytes("0f4f750c", |cb| cmovg(cb, ESI, mem_opnd(32, RBP, 12))); - check_bytes("0f4cc1", |cb| cmovl(cb, EAX, ECX)); - check_bytes("480f4cdd", |cb| cmovl(cb, RBX, RBP)); - check_bytes("0f4e742404", |cb| cmovle(cb, ESI, mem_opnd(32, RSP, 4))); + let cb1 = compile(|cb| cmovg(cb, ESI, EDI)); + let cb2 = compile(|cb| cmovg(cb, ESI, mem_opnd(32, RBP, 12))); + let cb3 = compile(|cb| cmovl(cb, EAX, ECX)); + let cb4 = compile(|cb| cmovl(cb, RBX, RBP)); + let cb5 = compile(|cb| cmovle(cb, ESI, mem_opnd(32, RSP, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: cmovg esi, edi + 0x0: cmovg esi, dword ptr [rbp + 0xc] + 0x0: cmovl eax, ecx + 0x0: cmovl rbx, rbp + 0x0: cmovle esi, dword ptr [rsp + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 0f4ff7 + 0f4f750c + 0f4cc1 + 480f4cdd + 0f4e742404 + "); } #[test] fn test_cmp() { - check_bytes("38d1", |cb| cmp(cb, CL, DL)); - check_bytes("39f9", |cb| cmp(cb, ECX, EDI)); - check_bytes("493b1424", |cb| cmp(cb, RDX, mem_opnd(64, R12, 0))); - check_bytes("4883f802", |cb| cmp(cb, RAX, imm_opnd(2))); - check_bytes("81f900000080", |cb| cmp(cb, ECX, uimm_opnd(0x8000_0000))); + let cb1 = compile(|cb| cmp(cb, CL, DL)); + let cb2 = compile(|cb| cmp(cb, ECX, EDI)); + let cb3 = compile(|cb| cmp(cb, RDX, mem_opnd(64, R12, 0))); + let cb4 = compile(|cb| cmp(cb, RAX, imm_opnd(2))); + let cb5 = compile(|cb| cmp(cb, ECX, uimm_opnd(0x8000_0000))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: cmp cl, dl + 0x0: cmp ecx, edi + 0x0: cmp rdx, qword ptr [r12] + 0x0: cmp rax, 2 + 0x0: cmp ecx, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 38d1 + 39f9 + 493b1424 + 4883f802 + 81f900000080 + "); } #[test] fn test_cqo() { - check_bytes("4899", |cb| cqo(cb)); + let cb = compile(cqo); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cqo"); + assert_snapshot!(cb.hexdump(), @"4899"); } #[test] fn test_imul() { - check_bytes("480fafc3", |cb| imul(cb, RAX, RBX)); - check_bytes("480faf10", |cb| imul(cb, RDX, mem_opnd(64, RAX, 0))); + let cb1 = compile(|cb| imul(cb, RAX, RBX)); + let cb2 = compile(|cb| imul(cb, RDX, mem_opnd(64, RAX, 0))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: imul rax, rbx + 0x0: imul rdx, qword ptr [rax] + "); - // Operands flipped for encoding since multiplication is commutative - check_bytes("480faf10", |cb| imul(cb, mem_opnd(64, RAX, 0), RDX)); + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 480fafc3 + 480faf10 + "); +} + +#[test] +#[should_panic] +fn test_imul_mem_reg() { + // imul doesn't have (Mem, Reg) encoding. Since multiplication is communicative, imul() could + // swap operands. However, x86_scratch_split may need to move the result to the output operand, + // which can be complicated if the assembler may sometimes change the result operand. + // So x86_scratch_split should be responsible for that swap, not the assembler. + compile(|cb| imul(cb, mem_opnd(64, RAX, 0), RDX)); } #[test] fn test_jge_label() { - check_bytes("0f8dfaffffff", |cb| { + let cb = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); jge_label(cb, label_idx); cb.link_labels(); }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jge 0"); + assert_snapshot!(cb.hexdump(), @"0f8dfaffffff"); } #[test] fn test_jmp_label() { // Forward jump - check_bytes("e900000000", |cb| { + let cb1 = compile(|cb| { let label_idx = cb.new_label("next".to_owned()); jmp_label(cb, label_idx); cb.write_label(label_idx); cb.link_labels(); }); - // Backwards jump - check_bytes("e9fbffffff", |cb| { + let cb2 = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); cb.write_label(label_idx); jmp_label(cb, label_idx); cb.link_labels(); }); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: jmp 5 + 0x0: jmp 0 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + e900000000 + e9fbffffff + "); } #[test] fn test_jmp_rm() { - check_bytes("41ffe4", |cb| jmp_rm(cb, R12)); + let cb = compile(|cb| jmp_rm(cb, R12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jmp r12"); + assert_snapshot!(cb.hexdump(), @"41ffe4"); } #[test] fn test_jo_label() { - check_bytes("0f80faffffff", |cb| { + let cb = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); jo_label(cb, label_idx); cb.link_labels(); }); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jo 0"); + assert_snapshot!(cb.hexdump(), @"0f80faffffff"); } #[test] fn test_lea() { - check_bytes("488d5108", |cb| lea(cb, RDX, mem_opnd(64, RCX, 8))); - check_bytes("488d0500000000", |cb| lea(cb, RAX, mem_opnd(8, RIP, 0))); - check_bytes("488d0505000000", |cb| lea(cb, RAX, mem_opnd(8, RIP, 5))); - check_bytes("488d3d05000000", |cb| lea(cb, RDI, mem_opnd(8, RIP, 5))); + let cb1 = compile(|cb| lea(cb, RDX, mem_opnd(64, RCX, 8))); + let cb2 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 0))); + let cb3 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 5))); + let cb4 = compile(|cb| lea(cb, RDI, mem_opnd(8, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @r" + 0x0: lea rdx, [rcx + 8] + 0x0: lea rax, [rip] + 0x0: lea rax, [rip + 5] + 0x0: lea rdi, [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @r" + 488d5108 + 488d0500000000 + 488d0505000000 + 488d3d05000000 + "); } #[test] fn test_mov() { - check_bytes("b807000000", |cb| mov(cb, EAX, imm_opnd(7))); - check_bytes("b8fdffffff", |cb| mov(cb, EAX, imm_opnd(-3))); - check_bytes("41bf03000000", |cb| mov(cb, R15, imm_opnd(3))); - check_bytes("89d8", |cb| mov(cb, EAX, EBX)); - check_bytes("89c8", |cb| mov(cb, EAX, ECX)); - check_bytes("8b9380000000", |cb| mov(cb, EDX, mem_opnd(32, RBX, 128))); - check_bytes("488b442404", |cb| mov(cb, RAX, mem_opnd(64, RSP, 4))); - + let cb01 = compile(|cb| mov(cb, EAX, imm_opnd(7))); + let cb02 = compile(|cb| mov(cb, EAX, imm_opnd(-3))); + let cb03 = compile(|cb| mov(cb, R15, imm_opnd(3))); + let cb04 = compile(|cb| mov(cb, EAX, EBX)); + let cb05 = compile(|cb| mov(cb, EAX, ECX)); + let cb06 = compile(|cb| mov(cb, EDX, mem_opnd(32, RBX, 128))); + let cb07 = compile(|cb| mov(cb, RAX, mem_opnd(64, RSP, 4))); // Test `mov rax, 3` => `mov eax, 3` optimization - check_bytes("41b834000000", |cb| mov(cb, R8, imm_opnd(0x34))); - check_bytes("49b80000008000000000", |cb| mov(cb, R8, imm_opnd(0x80000000))); - check_bytes("49b8ffffffffffffffff", |cb| mov(cb, R8, imm_opnd(-1))); - - check_bytes("b834000000", |cb| mov(cb, RAX, imm_opnd(0x34))); - check_bytes("48b8020000000000c0ff", |cb| mov(cb, RAX, imm_opnd(-18014398509481982))); - check_bytes("48b80000008000000000", |cb| mov(cb, RAX, imm_opnd(0x80000000))); - check_bytes("48b8ccffffffffffffff", |cb| mov(cb, RAX, imm_opnd(-52))); // yasm thinks this could use a dword immediate instead of qword - check_bytes("48b8ffffffffffffffff", |cb| mov(cb, RAX, imm_opnd(-1))); // yasm thinks this could use a dword immediate instead of qword - check_bytes("4488c9", |cb| mov(cb, CL, R9B)); - check_bytes("4889c3", |cb| mov(cb, RBX, RAX)); - check_bytes("4889df", |cb| mov(cb, RDI, RBX)); - check_bytes("40b60b", |cb| mov(cb, SIL, imm_opnd(11))); - - check_bytes("c60424fd", |cb| mov(cb, mem_opnd(8, RSP, 0), imm_opnd(-3))); - check_bytes("48c7470801000000", |cb| mov(cb, mem_opnd(64, RDI, 8), imm_opnd(1))); - //check_bytes("67c7400411000000", |cb| mov(cb, mem_opnd(32, EAX, 4), imm_opnd(0x34))); // We don't distinguish between EAX and RAX here - that's probably fine? - check_bytes("c7400411000000", |cb| mov(cb, mem_opnd(32, RAX, 4), imm_opnd(17))); - check_bytes("41895814", |cb| mov(cb, mem_opnd(32, R8, 20), EBX)); - check_bytes("4d8913", |cb| mov(cb, mem_opnd(64, R11, 0), R10)); - check_bytes("48c742f8f4ffffff", |cb| mov(cb, mem_opnd(64, RDX, -8), imm_opnd(-12))); + let cb08 = compile(|cb| mov(cb, R8, imm_opnd(0x34))); + let cb09 = compile(|cb| mov(cb, R8, imm_opnd(0x80000000))); + let cb10 = compile(|cb| mov(cb, R8, imm_opnd(-1))); + let cb11 = compile(|cb| mov(cb, RAX, imm_opnd(0x34))); + let cb12 = compile(|cb| mov(cb, RAX, imm_opnd(-18014398509481982))); + let cb13 = compile(|cb| mov(cb, RAX, imm_opnd(0x80000000))); + let cb14 = compile(|cb| mov(cb, RAX, imm_opnd(-52))); // yasm thinks this could use a dword immediate instead of qword + let cb15 = compile(|cb| mov(cb, RAX, imm_opnd(-1))); // yasm thinks this could use a dword immediate instead of qword + let cb16 = compile(|cb| mov(cb, CL, R9B)); + let cb17 = compile(|cb| mov(cb, RBX, RAX)); + let cb18 = compile(|cb| mov(cb, RDI, RBX)); + let cb19 = compile(|cb| mov(cb, SIL, imm_opnd(11))); + let cb20 = compile(|cb| mov(cb, mem_opnd(8, RSP, 0), imm_opnd(-3))); + let cb21 = compile(|cb| mov(cb, mem_opnd(64, RDI, 8), imm_opnd(1))); + //let cb = compile(|cb| mov(cb, mem_opnd(32, EAX, 4), imm_opnd(0x34))); // We don't distinguish between EAX and RAX here - that's probably fine? + let cb22 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), imm_opnd(17))); + let cb23 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), uimm_opnd(0x80000001))); + let cb24 = compile(|cb| mov(cb, mem_opnd(32, R8, 20), EBX)); + let cb25 = compile(|cb| mov(cb, mem_opnd(64, R11, 0), R10)); + let cb26 = compile(|cb| mov(cb, mem_opnd(64, RDX, -8), imm_opnd(-12))); + + assert_disasm_snapshot!(disasms!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26, + ), @r" + 0x0: mov eax, 7 + 0x0: mov eax, 0xfffffffd + 0x0: mov r15d, 3 + 0x0: mov eax, ebx + 0x0: mov eax, ecx + 0x0: mov edx, dword ptr [rbx + 0x80] + 0x0: mov rax, qword ptr [rsp + 4] + 0x0: mov r8d, 0x34 + 0x0: movabs r8, 0x80000000 + 0x0: movabs r8, 0xffffffffffffffff + 0x0: mov eax, 0x34 + 0x0: movabs rax, 0xffc0000000000002 + 0x0: movabs rax, 0x80000000 + 0x0: movabs rax, 0xffffffffffffffcc + 0x0: movabs rax, 0xffffffffffffffff + 0x0: mov cl, r9b + 0x0: mov rbx, rax + 0x0: mov rdi, rbx + 0x0: mov sil, 0xb + 0x0: mov byte ptr [rsp], 0xfd + 0x0: mov qword ptr [rdi + 8], 1 + 0x0: mov dword ptr [rax + 4], 0x11 + 0x0: mov dword ptr [rax + 4], 0x80000001 + 0x0: mov dword ptr [r8 + 0x14], ebx + 0x0: mov qword ptr [r11], r10 + 0x0: mov qword ptr [rdx - 8], 0xfffffffffffffff4 + "); + + assert_snapshot!(hexdumps!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26 + ), @r" + b807000000 + b8fdffffff + 41bf03000000 + 89d8 + 89c8 + 8b9380000000 + 488b442404 + 41b834000000 + 49b80000008000000000 + 49b8ffffffffffffffff + b834000000 + 48b8020000000000c0ff + 48b80000008000000000 + 48b8ccffffffffffffff + 48b8ffffffffffffffff + 4488c9 + 4889c3 + 4889df + 40b60b + c60424fd + 48c7470801000000 + c7400411000000 + c7400401000080 + 41895814 + 4d8913 + 48c742f8f4ffffff + "); } #[test] fn test_movabs() { - check_bytes("49b83400000000000000", |cb| movabs(cb, R8, 0x34)); - check_bytes("49b80000008000000000", |cb| movabs(cb, R8, 0x80000000)); + let cb1 = compile(|cb| movabs(cb, R8, 0x34)); + let cb2 = compile(|cb| movabs(cb, R8, 0x80000000)); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: movabs r8, 0x34 + 0x0: movabs r8, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 49b83400000000000000 + 49b80000008000000000 + "); } #[test] fn test_mov_unsigned() { // MOV AL, imm8 - check_bytes("b001", |cb| mov(cb, AL, uimm_opnd(1))); - check_bytes("b0ff", |cb| mov(cb, AL, uimm_opnd(u8::MAX.into()))); - + let cb01 = compile(|cb| mov(cb, AL, uimm_opnd(1))); + let cb02 = compile(|cb| mov(cb, AL, uimm_opnd(u8::MAX.into()))); // MOV AX, imm16 - check_bytes("66b80100", |cb| mov(cb, AX, uimm_opnd(1))); - check_bytes("66b8ffff", |cb| mov(cb, AX, uimm_opnd(u16::MAX.into()))); - + let cb03 = compile(|cb| mov(cb, AX, uimm_opnd(1))); + let cb04 = compile(|cb| mov(cb, AX, uimm_opnd(u16::MAX.into()))); // MOV EAX, imm32 - check_bytes("b801000000", |cb| mov(cb, EAX, uimm_opnd(1))); - check_bytes("b8ffffffff", |cb| mov(cb, EAX, uimm_opnd(u32::MAX.into()))); - check_bytes("41b800000000", |cb| mov(cb, R8, uimm_opnd(0))); - check_bytes("41b8ffffffff", |cb| mov(cb, R8, uimm_opnd(0xFF_FF_FF_FF))); - + let cb05 = compile(|cb| mov(cb, EAX, uimm_opnd(1))); + let cb06 = compile(|cb| mov(cb, EAX, uimm_opnd(u32::MAX.into()))); + let cb07 = compile(|cb| mov(cb, R8, uimm_opnd(0))); + let cb08 = compile(|cb| mov(cb, R8, uimm_opnd(0xFF_FF_FF_FF))); // MOV RAX, imm64, will move down into EAX since it fits into 32 bits - check_bytes("b801000000", |cb| mov(cb, RAX, uimm_opnd(1))); - check_bytes("b8ffffffff", |cb| mov(cb, RAX, uimm_opnd(u32::MAX.into()))); - + let cb09 = compile(|cb| mov(cb, RAX, uimm_opnd(1))); + let cb10 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX.into()))); // MOV RAX, imm64, will not move down into EAX since it does not fit into 32 bits - check_bytes("48b80000000001000000", |cb| mov(cb, RAX, uimm_opnd(u32::MAX as u64 + 1))); - check_bytes("48b8ffffffffffffffff", |cb| mov(cb, RAX, uimm_opnd(u64::MAX))); - check_bytes("49b8ffffffffffffffff", |cb| mov(cb, R8, uimm_opnd(u64::MAX))); - + let cb11 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX as u64 + 1))); + let cb12 = compile(|cb| mov(cb, RAX, uimm_opnd(u64::MAX))); + let cb13 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); // MOV r8, imm8 - check_bytes("41b001", |cb| mov(cb, R8B, uimm_opnd(1))); - check_bytes("41b0ff", |cb| mov(cb, R8B, uimm_opnd(u8::MAX.into()))); - + let cb14 = compile(|cb| mov(cb, R8B, uimm_opnd(1))); + let cb15 = compile(|cb| mov(cb, R8B, uimm_opnd(u8::MAX.into()))); // MOV r16, imm16 - check_bytes("6641b80100", |cb| mov(cb, R8W, uimm_opnd(1))); - check_bytes("6641b8ffff", |cb| mov(cb, R8W, uimm_opnd(u16::MAX.into()))); - + let cb16 = compile(|cb| mov(cb, R8W, uimm_opnd(1))); + let cb17 = compile(|cb| mov(cb, R8W, uimm_opnd(u16::MAX.into()))); // MOV r32, imm32 - check_bytes("41b801000000", |cb| mov(cb, R8D, uimm_opnd(1))); - check_bytes("41b8ffffffff", |cb| mov(cb, R8D, uimm_opnd(u32::MAX.into()))); - + let cb18 = compile(|cb| mov(cb, R8D, uimm_opnd(1))); + let cb19 = compile(|cb| mov(cb, R8D, uimm_opnd(u32::MAX.into()))); // MOV r64, imm64, will move down into 32 bit since it fits into 32 bits - check_bytes("41b801000000", |cb| mov(cb, R8, uimm_opnd(1))); - + let cb20 = compile(|cb| mov(cb, R8, uimm_opnd(1))); // MOV r64, imm64, will not move down into 32 bit since it does not fit into 32 bits - check_bytes("49b8ffffffffffffffff", |cb| mov(cb, R8, uimm_opnd(u64::MAX))); + let cb21 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @r" + 0x0: mov al, 1 + 0x0: mov al, 0xff + 0x0: mov ax, 1 + 0x0: mov ax, 0xffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: mov r8d, 0 + 0x0: mov r8d, 0xffffffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: movabs rax, 0x100000000 + 0x0: movabs rax, 0xffffffffffffffff + 0x0: movabs r8, 0xffffffffffffffff + 0x0: mov r8b, 1 + 0x0: mov r8b, 0xff + 0x0: mov r8w, 1 + 0x0: mov r8w, 0xffff + 0x0: mov r8d, 1 + 0x0: mov r8d, 0xffffffff + 0x0: mov r8d, 1 + 0x0: movabs r8, 0xffffffffffffffff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @r" + b001 + b0ff + 66b80100 + 66b8ffff + b801000000 + b8ffffffff + 41b800000000 + 41b8ffffffff + b801000000 + b8ffffffff + 48b80000000001000000 + 48b8ffffffffffffffff + 49b8ffffffffffffffff + 41b001 + 41b0ff + 6641b80100 + 6641b8ffff + 41b801000000 + 41b8ffffffff + 41b801000000 + 49b8ffffffffffffffff + "); } #[test] fn test_mov_iprel() { - check_bytes("8b0500000000", |cb| mov(cb, EAX, mem_opnd(32, RIP, 0))); - check_bytes("8b0505000000", |cb| mov(cb, EAX, mem_opnd(32, RIP, 5))); - - check_bytes("488b0500000000", |cb| mov(cb, RAX, mem_opnd(64, RIP, 0))); - check_bytes("488b0505000000", |cb| mov(cb, RAX, mem_opnd(64, RIP, 5))); - check_bytes("488b3d05000000", |cb| mov(cb, RDI, mem_opnd(64, RIP, 5))); + let cb1 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 0))); + let cb2 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 5))); + let cb3 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 0))); + let cb4 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 5))); + let cb5 = compile(|cb| mov(cb, RDI, mem_opnd(64, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: mov eax, dword ptr [rip] + 0x0: mov eax, dword ptr [rip + 5] + 0x0: mov rax, qword ptr [rip] + 0x0: mov rax, qword ptr [rip + 5] + 0x0: mov rdi, qword ptr [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 8b0500000000 + 8b0505000000 + 488b0500000000 + 488b0505000000 + 488b3d05000000 + "); } #[test] fn test_movsx() { - check_bytes("660fbec0", |cb| movsx(cb, AX, AL)); - check_bytes("0fbed0", |cb| movsx(cb, EDX, AL)); - check_bytes("480fbec3", |cb| movsx(cb, RAX, BL)); - check_bytes("0fbfc8", |cb| movsx(cb, ECX, AX)); - check_bytes("4c0fbed9", |cb| movsx(cb, R11, CL)); - check_bytes("4c6354240c", |cb| movsx(cb, R10, mem_opnd(32, RSP, 12))); - check_bytes("480fbe0424", |cb| movsx(cb, RAX, mem_opnd(8, RSP, 0))); - check_bytes("490fbf5504", |cb| movsx(cb, RDX, mem_opnd(16, R13, 4))); + let cb1 = compile(|cb| movsx(cb, AX, AL)); + let cb2 = compile(|cb| movsx(cb, EDX, AL)); + let cb3 = compile(|cb| movsx(cb, RAX, BL)); + let cb4 = compile(|cb| movsx(cb, ECX, AX)); + let cb5 = compile(|cb| movsx(cb, R11, CL)); + let cb6 = compile(|cb| movsx(cb, R10, mem_opnd(32, RSP, 12))); + let cb7 = compile(|cb| movsx(cb, RAX, mem_opnd(8, RSP, 0))); + let cb8 = compile(|cb| movsx(cb, RDX, mem_opnd(16, R13, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 0x0: movsx ax, al + 0x0: movsx edx, al + 0x0: movsx rax, bl + 0x0: movsx ecx, ax + 0x0: movsx r11, cl + 0x0: movsxd r10, dword ptr [rsp + 0xc] + 0x0: movsx rax, byte ptr [rsp] + 0x0: movsx rdx, word ptr [r13 + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 660fbec0 + 0fbed0 + 480fbec3 + 0fbfc8 + 4c0fbed9 + 4c6354240c + 480fbe0424 + 490fbf5504 + "); } #[test] fn test_nop() { - check_bytes("90", |cb| nop(cb, 1)); - check_bytes("6690", |cb| nop(cb, 2)); - check_bytes("0f1f00", |cb| nop(cb, 3)); - check_bytes("0f1f4000", |cb| nop(cb, 4)); - check_bytes("0f1f440000", |cb| nop(cb, 5)); - check_bytes("660f1f440000", |cb| nop(cb, 6)); - check_bytes("0f1f8000000000", |cb| nop(cb, 7)); - check_bytes("0f1f840000000000", |cb| nop(cb, 8)); - check_bytes("660f1f840000000000", |cb| nop(cb, 9)); - check_bytes("660f1f84000000000090", |cb| nop(cb, 10)); - check_bytes("660f1f8400000000006690", |cb| nop(cb, 11)); - check_bytes("660f1f8400000000000f1f00", |cb| nop(cb, 12)); + let cb01 = compile(|cb| nop(cb, 1)); + let cb02 = compile(|cb| nop(cb, 2)); + let cb03 = compile(|cb| nop(cb, 3)); + let cb04 = compile(|cb| nop(cb, 4)); + let cb05 = compile(|cb| nop(cb, 5)); + let cb06 = compile(|cb| nop(cb, 6)); + let cb07 = compile(|cb| nop(cb, 7)); + let cb08 = compile(|cb| nop(cb, 8)); + let cb09 = compile(|cb| nop(cb, 9)); + let cb10 = compile(|cb| nop(cb, 10)); + let cb11 = compile(|cb| nop(cb, 11)); + let cb12 = compile(|cb| nop(cb, 12)); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @r" + 0x0: nop + 0x0: nop + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop dword ptr [rax] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @r" + 90 + 6690 + 0f1f00 + 0f1f4000 + 0f1f440000 + 660f1f440000 + 0f1f8000000000 + 0f1f840000000000 + 660f1f840000000000 + 660f1f84000000000090 + 660f1f8400000000006690 + 660f1f8400000000000f1f00 + "); } #[test] fn test_not() { - check_bytes("66f7d0", |cb| not(cb, AX)); - check_bytes("f7d0", |cb| not(cb, EAX)); - check_bytes("49f71424", |cb| not(cb, mem_opnd(64, R12, 0))); - check_bytes("f794242d010000", |cb| not(cb, mem_opnd(32, RSP, 301))); - check_bytes("f71424", |cb| not(cb, mem_opnd(32, RSP, 0))); - check_bytes("f7542403", |cb| not(cb, mem_opnd(32, RSP, 3))); - check_bytes("f75500", |cb| not(cb, mem_opnd(32, RBP, 0))); - check_bytes("f7550d", |cb| not(cb, mem_opnd(32, RBP, 13))); - check_bytes("48f7d0", |cb| not(cb, RAX)); - check_bytes("49f7d3", |cb| not(cb, R11)); - check_bytes("f710", |cb| not(cb, mem_opnd(32, RAX, 0))); - check_bytes("f716", |cb| not(cb, mem_opnd(32, RSI, 0))); - check_bytes("f717", |cb| not(cb, mem_opnd(32, RDI, 0))); - check_bytes("f75237", |cb| not(cb, mem_opnd(32, RDX, 55))); - check_bytes("f79239050000", |cb| not(cb, mem_opnd(32, RDX, 1337))); - check_bytes("f752c9", |cb| not(cb, mem_opnd(32, RDX, -55))); - check_bytes("f792d5fdffff", |cb| not(cb, mem_opnd(32, RDX, -555))); + let cb01 = compile(|cb| not(cb, AX)); + let cb02 = compile(|cb| not(cb, EAX)); + let cb03 = compile(|cb| not(cb, mem_opnd(64, R12, 0))); + let cb04 = compile(|cb| not(cb, mem_opnd(32, RSP, 301))); + let cb05 = compile(|cb| not(cb, mem_opnd(32, RSP, 0))); + let cb06 = compile(|cb| not(cb, mem_opnd(32, RSP, 3))); + let cb07 = compile(|cb| not(cb, mem_opnd(32, RBP, 0))); + let cb08 = compile(|cb| not(cb, mem_opnd(32, RBP, 13))); + let cb09 = compile(|cb| not(cb, RAX)); + let cb10 = compile(|cb| not(cb, R11)); + let cb11 = compile(|cb| not(cb, mem_opnd(32, RAX, 0))); + let cb12 = compile(|cb| not(cb, mem_opnd(32, RSI, 0))); + let cb13 = compile(|cb| not(cb, mem_opnd(32, RDI, 0))); + let cb14 = compile(|cb| not(cb, mem_opnd(32, RDX, 55))); + let cb15 = compile(|cb| not(cb, mem_opnd(32, RDX, 1337))); + let cb16 = compile(|cb| not(cb, mem_opnd(32, RDX, -55))); + let cb17 = compile(|cb| not(cb, mem_opnd(32, RDX, -555))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @r" + 0x0: not ax + 0x0: not eax + 0x0: not qword ptr [r12] + 0x0: not dword ptr [rsp + 0x12d] + 0x0: not dword ptr [rsp] + 0x0: not dword ptr [rsp + 3] + 0x0: not dword ptr [rbp] + 0x0: not dword ptr [rbp + 0xd] + 0x0: not rax + 0x0: not r11 + 0x0: not dword ptr [rax] + 0x0: not dword ptr [rsi] + 0x0: not dword ptr [rdi] + 0x0: not dword ptr [rdx + 0x37] + 0x0: not dword ptr [rdx + 0x539] + 0x0: not dword ptr [rdx - 0x37] + 0x0: not dword ptr [rdx - 0x22b] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @r" + 66f7d0 + f7d0 + 49f71424 + f794242d010000 + f71424 + f7542403 + f75500 + f7550d + 48f7d0 + 49f7d3 + f710 + f716 + f717 + f75237 + f79239050000 + f752c9 + f792d5fdffff + "); } #[test] fn test_or() { - check_bytes("09f2", |cb| or(cb, EDX, ESI)); + let cb = compile(|cb| or(cb, EDX, ESI)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or edx, esi"); + assert_snapshot!(cb.hexdump(), @"09f2"); } #[test] fn test_pop() { - check_bytes("58", |cb| pop(cb, RAX)); - check_bytes("5b", |cb| pop(cb, RBX)); - check_bytes("5c", |cb| pop(cb, RSP)); - check_bytes("5d", |cb| pop(cb, RBP)); - check_bytes("415c", |cb| pop(cb, R12)); - check_bytes("8f00", |cb| pop(cb, mem_opnd(64, RAX, 0))); - check_bytes("418f00", |cb| pop(cb, mem_opnd(64, R8, 0))); - check_bytes("418f4003", |cb| pop(cb, mem_opnd(64, R8, 3))); - check_bytes("8f44c803", |cb| pop(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); - check_bytes("418f44c803", |cb| pop(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + let cb01 = compile(|cb| pop(cb, RAX)); + let cb02 = compile(|cb| pop(cb, RBX)); + let cb03 = compile(|cb| pop(cb, RSP)); + let cb04 = compile(|cb| pop(cb, RBP)); + let cb05 = compile(|cb| pop(cb, R12)); + let cb06 = compile(|cb| pop(cb, mem_opnd(64, RAX, 0))); + let cb07 = compile(|cb| pop(cb, mem_opnd(64, R8, 0))); + let cb08 = compile(|cb| pop(cb, mem_opnd(64, R8, 3))); + let cb09 = compile(|cb| pop(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb10 = compile(|cb| pop(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @r" + 0x0: pop rax + 0x0: pop rbx + 0x0: pop rsp + 0x0: pop rbp + 0x0: pop r12 + 0x0: pop qword ptr [rax] + 0x0: pop qword ptr [r8] + 0x0: pop qword ptr [r8 + 3] + 0x0: pop qword ptr [rax + rcx*8 + 3] + 0x0: pop qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @r" + 58 + 5b + 5c + 5d + 415c + 8f00 + 418f00 + 418f4003 + 8f44c803 + 418f44c803 + "); } #[test] fn test_push() { - check_bytes("50", |cb| push(cb, RAX)); - check_bytes("53", |cb| push(cb, RBX)); - check_bytes("4154", |cb| push(cb, R12)); - check_bytes("ff30", |cb| push(cb, mem_opnd(64, RAX, 0))); - check_bytes("41ff30", |cb| push(cb, mem_opnd(64, R8, 0))); - check_bytes("41ff7003", |cb| push(cb, mem_opnd(64, R8, 3))); - check_bytes("ff74c803", |cb| push(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); - check_bytes("41ff74c803", |cb| push(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + let cb1 = compile(|cb| push(cb, RAX)); + let cb2 = compile(|cb| push(cb, RBX)); + let cb3 = compile(|cb| push(cb, R12)); + let cb4 = compile(|cb| push(cb, mem_opnd(64, RAX, 0))); + let cb5 = compile(|cb| push(cb, mem_opnd(64, R8, 0))); + let cb6 = compile(|cb| push(cb, mem_opnd(64, R8, 3))); + let cb7 = compile(|cb| push(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb8 = compile(|cb| push(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 0x0: push rax + 0x0: push rbx + 0x0: push r12 + 0x0: push qword ptr [rax] + 0x0: push qword ptr [r8] + 0x0: push qword ptr [r8 + 3] + 0x0: push qword ptr [rax + rcx*8 + 3] + 0x0: push qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @r" + 50 + 53 + 4154 + ff30 + 41ff30 + 41ff7003 + ff74c803 + 41ff74c803 + "); } #[test] fn test_ret() { - check_bytes("c3", |cb| ret(cb)); + let cb = compile(ret); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c3"); } #[test] fn test_sal() { - check_bytes("66d1e1", |cb| sal(cb, CX, uimm_opnd(1))); - check_bytes("d1e1", |cb| sal(cb, ECX, uimm_opnd(1))); - check_bytes("c1e505", |cb| sal(cb, EBP, uimm_opnd(5))); - check_bytes("d1642444", |cb| sal(cb, mem_opnd(32, RSP, 68), uimm_opnd(1))); - check_bytes("48d3e1", |cb| sal(cb, RCX, CL)); + let cb1 = compile(|cb| sal(cb, CX, uimm_opnd(1))); + let cb2 = compile(|cb| sal(cb, ECX, uimm_opnd(1))); + let cb3 = compile(|cb| sal(cb, EBP, uimm_opnd(5))); + let cb4 = compile(|cb| sal(cb, mem_opnd(32, RSP, 68), uimm_opnd(1))); + let cb5 = compile(|cb| sal(cb, RCX, CL)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @r" + 0x0: shl cx, 1 + 0x0: shl ecx, 1 + 0x0: shl ebp, 5 + 0x0: shl dword ptr [rsp + 0x44], 1 + 0x0: shl rcx, cl + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @r" + 66d1e1 + d1e1 + c1e505 + d1642444 + 48d3e1 + "); } #[test] fn test_sar() { - check_bytes("d1fa", |cb| sar(cb, EDX, uimm_opnd(1))); + let cb = compile(|cb| sar(cb, EDX, uimm_opnd(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sar edx, 1"); + assert_snapshot!(cb.hexdump(), @"d1fa"); } #[test] fn test_shr() { - check_bytes("49c1ee07", |cb| shr(cb, R14, uimm_opnd(7))); + let cb = compile(|cb| shr(cb, R14, uimm_opnd(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: shr r14, 7"); + assert_snapshot!(cb.hexdump(), @"49c1ee07"); } #[test] fn test_sub() { - check_bytes("83e801", |cb| sub(cb, EAX, imm_opnd(1))); - check_bytes("4883e802", |cb| sub(cb, RAX, imm_opnd(2))); + let cb1 = compile(|cb| sub(cb, EAX, imm_opnd(1))); + let cb2 = compile(|cb| sub(cb, RAX, imm_opnd(2))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @r" + 0x0: sub eax, 1 + 0x0: sub rax, 2 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @r" + 83e801 + 4883e802 + "); } #[test] @@ -374,44 +842,103 @@ fn test_sub() { fn test_sub_uimm_too_large() { // This immediate becomes a different value after // sign extension, so not safe to encode. - check_bytes("ff", |cb| sub(cb, RCX, uimm_opnd(0x8000_0000))); + compile(|cb| sub(cb, RCX, uimm_opnd(0x8000_0000))); } #[test] fn test_test() { - check_bytes("84c0", |cb| test(cb, AL, AL)); - check_bytes("6685c0", |cb| test(cb, AX, AX)); - check_bytes("f6c108", |cb| test(cb, CL, uimm_opnd(8))); - check_bytes("f6c207", |cb| test(cb, DL, uimm_opnd(7))); - check_bytes("f6c108", |cb| test(cb, RCX, uimm_opnd(8))); - check_bytes("f6420808", |cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(8))); - check_bytes("f64208ff", |cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(255))); - check_bytes("66f7c2ffff", |cb| test(cb, DX, uimm_opnd(0xffff))); - check_bytes("66f74208ffff", |cb| test(cb, mem_opnd(16, RDX, 8), uimm_opnd(0xffff))); - check_bytes("f60601", |cb| test(cb, mem_opnd(8, RSI, 0), uimm_opnd(1))); - check_bytes("f6461001", |cb| test(cb, mem_opnd(8, RSI, 16), uimm_opnd(1))); - check_bytes("f646f001", |cb| test(cb, mem_opnd(8, RSI, -16), uimm_opnd(1))); - check_bytes("854640", |cb| test(cb, mem_opnd(32, RSI, 64), EAX)); - check_bytes("4885472a", |cb| test(cb, mem_opnd(64, RDI, 42), RAX)); - check_bytes("4885c0", |cb| test(cb, RAX, RAX)); - check_bytes("4885f0", |cb| test(cb, RAX, RSI)); - check_bytes("48f74640f7ffffff", |cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08))); - check_bytes("48f7464008000000", |cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(0x08))); - check_bytes("48f7c108000000", |cb| test(cb, RCX, imm_opnd(0x08))); - //check_bytes("48a9f7ffff0f", |cb| test(cb, RAX, imm_opnd(0x0FFFFFF7))); + let cb01 = compile(|cb| test(cb, AL, AL)); + let cb02 = compile(|cb| test(cb, AX, AX)); + let cb03 = compile(|cb| test(cb, CL, uimm_opnd(8))); + let cb04 = compile(|cb| test(cb, DL, uimm_opnd(7))); + let cb05 = compile(|cb| test(cb, RCX, uimm_opnd(8))); + let cb06 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(8))); + let cb07 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(255))); + let cb08 = compile(|cb| test(cb, DX, uimm_opnd(0xffff))); + let cb09 = compile(|cb| test(cb, mem_opnd(16, RDX, 8), uimm_opnd(0xffff))); + let cb10 = compile(|cb| test(cb, mem_opnd(8, RSI, 0), uimm_opnd(1))); + let cb11 = compile(|cb| test(cb, mem_opnd(8, RSI, 16), uimm_opnd(1))); + let cb12 = compile(|cb| test(cb, mem_opnd(8, RSI, -16), uimm_opnd(1))); + let cb13 = compile(|cb| test(cb, mem_opnd(32, RSI, 64), EAX)); + let cb14 = compile(|cb| test(cb, mem_opnd(64, RDI, 42), RAX)); + let cb15 = compile(|cb| test(cb, RAX, RAX)); + let cb16 = compile(|cb| test(cb, RAX, RSI)); + let cb17 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08))); + let cb18 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(0x08))); + let cb19 = compile(|cb| test(cb, RCX, imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @r" + 0x0: test al, al + 0x0: test ax, ax + 0x0: test cl, 8 + 0x0: test dl, 7 + 0x0: test cl, 8 + 0x0: test byte ptr [rdx + 8], 8 + 0x0: test byte ptr [rdx + 8], 0xff + 0x0: test dx, 0xffff + 0x0: test word ptr [rdx + 8], 0xffff + 0x0: test byte ptr [rsi], 1 + 0x0: test byte ptr [rsi + 0x10], 1 + 0x0: test byte ptr [rsi - 0x10], 1 + 0x0: test dword ptr [rsi + 0x40], eax + 0x0: test qword ptr [rdi + 0x2a], rax + 0x0: test rax, rax + 0x0: test rax, rsi + 0x0: test qword ptr [rsi + 0x40], -9 + 0x0: test qword ptr [rsi + 0x40], 8 + 0x0: test rcx, 8 + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @r" + 84c0 + 6685c0 + f6c108 + f6c207 + f6c108 + f6420808 + f64208ff + 66f7c2ffff + 66f74208ffff + f60601 + f6461001 + f646f001 + 854640 + 4885472a + 4885c0 + 4885f0 + 48f74640f7ffffff + 48f7464008000000 + 48f7c108000000 + "); } #[test] fn test_xchg() { - check_bytes("4891", |cb| xchg(cb, RAX, RCX)); - check_bytes("4995", |cb| xchg(cb, RAX, R13)); - check_bytes("4887d9", |cb| xchg(cb, RCX, RBX)); - check_bytes("4d87f9", |cb| xchg(cb, R9, R15)); + let cb1 = compile(|cb| xchg(cb, RAX, RCX)); + let cb2 = compile(|cb| xchg(cb, RAX, R13)); + let cb3 = compile(|cb| xchg(cb, RCX, RBX)); + let cb4 = compile(|cb| xchg(cb, R9, R15)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @r" + 0x0: xchg rcx, rax + 0x0: xchg r13, rax + 0x0: xchg rcx, rbx + 0x0: xchg r9, r15 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @r" + 4891 + 4995 + 4887d9 + 4d87f9 + "); } #[test] fn test_xor() { - check_bytes("31c0", |cb| xor(cb, EAX, EAX)); + let cb = compile(|cb| xor(cb, EAX, EAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor eax, eax"); + assert_snapshot!(cb.hexdump(), @"31c0"); } #[test] @@ -437,25 +964,3 @@ fn basic_capstone_usage() -> std::result::Result<(), capstone::Error> { )), } } - -#[test] -#[cfg(feature = "disasm")] -fn block_comments() { - let mut cb = super::CodeBlock::new_dummy(4096); - - let first_write_ptr = cb.get_write_ptr().raw_addr(&cb); - cb.add_comment("Beginning"); - xor(&mut cb, EAX, EAX); // 2 bytes long - let second_write_ptr = cb.get_write_ptr().raw_addr(&cb); - cb.add_comment("Two bytes in"); - cb.add_comment("Still two bytes in"); - cb.add_comment("Still two bytes in"); // Duplicate, should be ignored - test(&mut cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08)); // 8 bytes long - let third_write_ptr = cb.get_write_ptr().raw_addr(&cb); - cb.add_comment("Ten bytes in"); - - assert_eq!(&vec!( "Beginning".to_string() ), cb.comments_at(first_write_ptr).unwrap()); - assert_eq!(&vec!( "Two bytes in".to_string(), "Still two bytes in".to_string() ), cb.comments_at(second_write_ptr).unwrap()); - assert_eq!(&vec!( "Ten bytes in".to_string() ), cb.comments_at(third_write_ptr).unwrap()); -} -*/ diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index ffde567b69..55a65e3ea6 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -2,21 +2,29 @@ use std::mem::take; use crate::asm::{CodeBlock, Label}; use crate::asm::arm64::*; +use crate::codegen::split_patch_point; use crate::cruby::*; use crate::backend::lir::*; +use crate::options::asm_dump; +use crate::stats::CompileError; use crate::virtualmem::CodePtr; use crate::cast::*; // Use the arm64 register type for this platform pub type Reg = A64Reg; +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_no } +} + // Callee-saved registers -pub const _CFP: Opnd = Opnd::Reg(X19_REG); -pub const _EC: Opnd = Opnd::Reg(X20_REG); -pub const _SP: Opnd = Opnd::Reg(X21_REG); +pub const CFP: Opnd = Opnd::Reg(X19_REG); +pub const EC: Opnd = Opnd::Reg(X20_REG); +pub const SP: Opnd = Opnd::Reg(X21_REG); // C argument registers on this platform -pub const _C_ARG_OPNDS: [Opnd; 6] = [ +pub const C_ARG_OPNDS: [Opnd; 6] = [ Opnd::Reg(X0_REG), Opnd::Reg(X1_REG), Opnd::Reg(X2_REG), @@ -27,7 +35,9 @@ pub const _C_ARG_OPNDS: [Opnd; 6] = [ // C return value register on this platform pub const C_RET_REG: Reg = X0_REG; -pub const _C_RET_OPND: Opnd = Opnd::Reg(X0_REG); +pub const C_RET_OPND: Opnd = Opnd::Reg(X0_REG); +pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(XZR_REG); +pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(X29_REG); // These constants define the way we work with Arm64's stack pointer. The stack // pointer always needs to be aligned to a 16-byte boundary. @@ -70,12 +80,14 @@ impl From<Opnd> for A64Opnd { Opnd::Mem(Mem { base: MemBase::VReg(_), .. }) => { panic!("attempted to lower an Opnd::Mem with a MemBase::VReg base") }, + Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. }) => { + panic!("attempted to lower an Opnd::Mem with a MemBase::Stack base") + }, Opnd::VReg { .. } => panic!("attempted to lower an Opnd::VReg"), Opnd::Value(_) => panic!("attempted to lower an Opnd::Value"), Opnd::None => panic!( "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." ), - } } } @@ -96,7 +108,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { let start = cb.get_write_ptr(); emit_jmp_ptr(cb, dst_ptr, true); let end = cb.get_write_ptr(); - unsafe { rb_zjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; } fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { @@ -111,8 +123,8 @@ fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { b(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); 1 } else { - let num_insns = emit_load_value(cb, Assembler::SCRATCH0, dst_addr as u64); - br(cb, Assembler::SCRATCH0); + let num_insns = emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); num_insns + 1 }; @@ -137,13 +149,17 @@ fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { // If the value fits into a single movz // instruction, then we'll use that. movz(cb, rd, A64Opnd::new_uimm(current), 0); - return 1; + 1 + } else if u16::try_from(!value).is_ok() { + // For small negative values, use a single movn + movn(cb, rd, A64Opnd::new_uimm(!value), 0); + 1 } else if BitmaskImmediate::try_from(current).is_ok() { // Otherwise, if the immediate can be encoded // with the special bitmask immediate encoding, // we'll use that. mov(cb, rd, A64Opnd::new_uimm(current)); - return 1; + 1 } else { // Finally we'll fall back to encoding the value // using movz for the first 16 bits and movk for @@ -169,14 +185,14 @@ fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 48); num_insns += 1; } - return num_insns; + num_insns } } /// List of registers that can be used for register allocation. /// This has the same number of registers for x86_64 and arm64. -/// SCRATCH0 and SCRATCH1 are excluded. -pub const ALLOC_REGS: &'static [Reg] = &[ +/// SCRATCH_OPND, SCRATCH1_OPND, and EMIT_OPND are excluded. +pub const ALLOC_REGS: &[Reg] = &[ X0_REG, X1_REG, X2_REG, @@ -187,19 +203,26 @@ pub const ALLOC_REGS: &'static [Reg] = &[ X12_REG, ]; -#[derive(Debug, PartialEq)] -enum EmitError { - RetryOnNextPage, - OutOfMemory, -} +/// Special scratch registers for intermediate processing. They should be used only by +/// [`Assembler::arm64_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(X15_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(X17_REG); +const SCRATCH2_OPND: Opnd = Opnd::Reg(X14_REG); -impl Assembler -{ - // Special scratch registers for intermediate processing. - // This register is caller-saved (so we don't have to save it before using it) - pub const SCRATCH_REG: Reg = X16_REG; - const SCRATCH0: A64Opnd = A64Opnd::Reg(Assembler::SCRATCH_REG); - const SCRATCH1: A64Opnd = A64Opnd::Reg(X17_REG); +impl Assembler { + /// Special register for intermediate processing in arm64_emit. It should be used only by arm64_emit. + const EMIT_REG: Reg = X16_REG; + const EMIT_OPND: A64Opnd = A64Opnd::Reg(Self::EMIT_REG); + + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } /// Get the list of registers from which we will allocate on this platform pub fn get_alloc_regs() -> Vec<Reg> { @@ -211,6 +234,11 @@ impl Assembler vec![X1_REG, X9_REG, X10_REG, X11_REG, X12_REG, X13_REG, X14_REG, X15_REG] } + /// How many bytes a call and a [Self::frame_setup] would change native SP + pub fn frame_size() -> i32 { + 0x10 + } + /// Split platform-specific instructions /// The transformations done here are meant to make our lives simpler in later /// stages of the compilation pipeline. @@ -218,29 +246,6 @@ impl Assembler /// have no memory operands. fn arm64_split(mut self) -> Assembler { - /// When we're attempting to load a memory address into a register, the - /// displacement must fit into the maximum number of bits for an Op::Add - /// immediate. If it doesn't, we have to load the displacement into a - /// register first. - fn split_lea_operand(asm: &mut Assembler, opnd: Opnd) -> Opnd { - match opnd { - Opnd::Mem(Mem { base, disp, num_bits }) => { - if disp >= 0 && ShiftedImmediate::try_from(disp as u64).is_ok() { - asm.lea(opnd) - } else { - let disp = asm.load(Opnd::Imm(disp.into())); - let reg = match base { - MemBase::Reg(reg_no) => Opnd::Reg(Reg { reg_no, num_bits }), - MemBase::VReg(idx) => Opnd::VReg { idx, num_bits } - }; - - asm.add(reg, disp) - } - }, - _ => unreachable!("Op::Lea only accepts Opnd::Mem operands.") - } - } - /// When you're storing a register into a memory location or loading a /// memory location into a register, the displacement from the base /// register of the memory location must fit into 9 bits. If it doesn't, @@ -251,8 +256,8 @@ impl Assembler if mem_disp_fits_bits(mem.disp) { opnd } else { - let base = split_lea_operand(asm, opnd); - Opnd::mem(64, base, 0) + let base = asm.lea(Opnd::Mem(Mem { num_bits: 64, ..mem })); + Opnd::mem(mem.num_bits, base, 0) } }, _ => unreachable!("Can only split memory addresses.") @@ -270,7 +275,7 @@ impl Assembler // Many Arm insns support only 32-bit or 64-bit operands. asm.load with fewer // bits zero-extends the value, so it's safe to recognize it as a 32-bit value. if out_opnd.rm_num_bits() < 32 { - out_opnd.with_num_bits(32).unwrap() + out_opnd.with_num_bits(32) } else { out_opnd } @@ -296,7 +301,7 @@ impl Assembler BitmaskImmediate::new_32b_reg(imm as u32).is_ok()) { Opnd::UImm(imm as u64) } else { - asm.load(opnd).with_num_bits(dest_num_bits).unwrap() + asm.load(opnd).with_num_bits(dest_num_bits) } }, Opnd::UImm(uimm) => { @@ -306,7 +311,7 @@ impl Assembler BitmaskImmediate::new_32b_reg(uimm as u32).is_ok()) { opnd } else { - asm.load(opnd).with_num_bits(dest_num_bits).unwrap() + asm.load(opnd).with_num_bits(dest_num_bits) } }, Opnd::None | Opnd::Value(_) => unreachable!() @@ -331,7 +336,7 @@ impl Assembler asm.load(opnd) } }, - Opnd::None | Opnd::Value(_) /*| Opnd::Stack { .. }*/ => unreachable!() + Opnd::None | Opnd::Value(_) => unreachable!() } } @@ -374,8 +379,8 @@ impl Assembler match opnd0 { Opnd::Reg(_) | Opnd::VReg { .. } => { match opnd0.rm_num_bits() { - 8 => asm.and(opnd0.with_num_bits(64).unwrap(), Opnd::UImm(0xff)), - 16 => asm.and(opnd0.with_num_bits(64).unwrap(), Opnd::UImm(0xffff)), + 8 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xff)), + 16 => asm.and(opnd0.with_num_bits(64), Opnd::UImm(0xffff)), 32 | 64 => opnd0, bits => unreachable!("Invalid number of bits. {}", bits) } @@ -384,12 +389,12 @@ impl Assembler } } + let mut asm_local = Assembler::new_with_asm(&self); let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); - let mut iterator = self.insns.into_iter().enumerate().peekable(); - let mut asm_local = Assembler::new_with_label_names(take(&mut self.label_names), live_ranges.len()); + let mut iterator = self.instruction_iterator(); let asm = &mut asm_local; - while let Some((index, mut insn)) = iterator.next() { + while let Some((index, mut insn)) = iterator.next(asm) { // Here we're going to map the operands of the instruction to load // any Opnd::Value operands into registers if they are heap objects // such that only the Op::Load instruction needs to handle that @@ -399,15 +404,12 @@ impl Assembler let mut opnd_iter = insn.opnd_iter_mut(); while let Some(opnd) = opnd_iter.next() { - match opnd { - Opnd::Value(value) => { - if value.special_const_p() { - *opnd = Opnd::UImm(value.as_u64()); - } else if !is_load { - *opnd = asm.load(*opnd); - } - }, - _ => {} + if let Opnd::Value(value) = opnd { + if value.special_const_p() { + *opnd = Opnd::UImm(value.as_u64()); + } else if !is_load { + *opnd = asm.load(*opnd); + } }; } @@ -415,24 +417,36 @@ impl Assembler // being used. It is okay not to use their output here. #[allow(unused_must_use)] match &mut insn { - Insn::Add { left, right, .. } => { + Insn::Add { left, right, out } => { match (*left, *right) { - (Opnd::Reg(_) | Opnd::VReg { .. }, Opnd::Reg(_) | Opnd::VReg { .. }) => { - asm.push_insn(insn); - }, + // When one operand is a register, legalize the other operand + // into possibly an immdiate and swap the order if necessary. + // Only the rhs of ADD can be an immediate, but addition is commutative. (reg_opnd @ (Opnd::Reg(_) | Opnd::VReg { .. }), other_opnd) | (other_opnd, reg_opnd @ (Opnd::Reg(_) | Opnd::VReg { .. })) => { *left = reg_opnd; *right = split_shifted_immediate(asm, other_opnd); + // Now `right` is either a register or an immediate, both can try to + // merge with a subsequent mov. + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, left, out); asm.push_insn(insn); - }, + } _ => { *left = split_load_operand(asm, *left); *right = split_shifted_immediate(asm, *right); + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, right, out); asm.push_insn(insn); } } - }, + } + Insn::Sub { left, right, out } => { + *left = split_load_operand(asm, *left); + *right = split_shifted_immediate(asm, *right); + // Now `right` is either a register or an immediate, + // both can try to merge with a subsequent mov. + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, left, out); + asm.push_insn(insn); + } Insn::And { left, right, out } | Insn::Or { left, right, out } | Insn::Xor { left, right, out } => { @@ -440,18 +454,7 @@ impl Assembler *left = opnd0; *right = opnd1; - // Since these instructions are lowered to an instruction that have 2 input - // registers and an output register, look to merge with an `Insn::Mov` that - // follows which puts the output in another register. For example: - // `Add a, b => out` followed by `Mov c, out` becomes `Add a, b => c`. - if let (Opnd::Reg(_), Opnd::Reg(_), Some(Insn::Mov { dest, src })) = (left, right, iterator.peek().map(|(_, insn)| insn)) { - if live_ranges[out.vreg_idx()].end() == index + 1 { - if out == src && matches!(*dest, Opnd::Reg(_)) { - *out = *dest; - iterator.next(); // Pop merged Insn::Mov - } - } - } + merge_three_reg_mov(&live_ranges, &mut iterator, asm, left, right, out); asm.push_insn(insn); } @@ -492,8 +495,8 @@ impl Assembler // Note: the iteration order is reversed to avoid corrupting x0, // which is both the return value and first argument register if !opnds.is_empty() { - let mut args: Vec<(Reg, Opnd)> = vec![]; - for (idx, opnd) in opnds.into_iter().enumerate().rev() { + let mut args: Vec<(Opnd, Opnd)> = vec![]; + for (idx, opnd) in opnds.iter_mut().enumerate().rev() { // If the value that we're sending is 0, then we can use // the zero register, so in this case we'll just send // a UImm of 0 along as the argument to the move. @@ -502,7 +505,7 @@ impl Assembler Opnd::Mem(_) => split_memory_address(asm, *opnd), _ => *opnd }; - args.push((C_ARG_OPNDS[idx].unwrap_reg(), value)); + args.push((C_ARG_OPNDS[idx], value)); } asm.parallel_mov(args); } @@ -518,7 +521,7 @@ impl Assembler let split_right = split_shifted_immediate(asm, *right); let opnd1 = match split_right { Opnd::VReg { .. } if opnd0.num_bits() != split_right.num_bits() => { - split_right.with_num_bits(opnd0.num_bits().unwrap()).unwrap() + split_right.with_num_bits(opnd0.num_bits().unwrap()) }, _ => split_right }; @@ -564,21 +567,13 @@ impl Assembler if matches!(out, Opnd::VReg { .. }) && *out == *src && live_ranges[out.vreg_idx()].end() == index + 1 => { *out = Opnd::Reg(*reg); asm.push_insn(insn); - iterator.next(); // Pop merged Insn::Mov + iterator.next(asm); // Pop merged Insn::Mov } _ => { asm.push_insn(insn); } } }, - Insn::IncrCounter { mem, value } => { - let counter_addr = match mem { - Opnd::Mem(_) => split_lea_operand(asm, *mem), - _ => *mem - }; - - asm.incr_counter(counter_addr, *value); - }, Insn::JmpOpnd(opnd) => { if let Opnd::Mem(_) = opnd { let opnd0 = split_load_operand(asm, *opnd); @@ -636,7 +631,7 @@ impl Assembler }, // If we're loading a memory operand into a register, then // we'll switch over to the load instruction. - (Opnd::Reg(_), Opnd::Mem(_)) => { + (Opnd::Reg(_) | Opnd::VReg { .. }, Opnd::Mem(_)) => { let value = split_memory_address(asm, *src); asm.load_into(*dest, value); }, @@ -649,7 +644,7 @@ impl Assembler }; asm.mov(*dest, value); }, - _ => unreachable!() + _ => unreachable!("unexpected combination of operands in Insn::Mov: {dest:?}, {src:?}") }; }, Insn::Not { opnd, .. } => { @@ -666,42 +661,7 @@ impl Assembler Insn::URShift { opnd, .. } => { // The operand must be in a register, so // if we get anything else we need to load it first. - let opnd0 = match opnd { - Opnd::Mem(_) => split_load_operand(asm, *opnd), - _ => *opnd - }; - - *opnd = opnd0; - asm.push_insn(insn); - }, - Insn::Store { dest, src } => { - // The value being stored must be in a register, so if it's - // not already one we'll load it first. - let opnd1 = match src { - // If the first operand is zero, then we can just use - // the zero register. - Opnd::UImm(0) | Opnd::Imm(0) => Opnd::Reg(XZR_REG), - // Otherwise we'll check if we need to load it first. - _ => split_load_operand(asm, *src) - }; - - match dest { - Opnd::Reg(_) => { - // Store does not support a register as a dest operand. - asm.mov(*dest, opnd1); - } - _ => { - // The displacement for the STUR instruction can't be more - // than 9 bits long. If it's longer, we need to load the - // memory address into a register first. - let opnd0 = split_memory_address(asm, *dest); - asm.store(opnd0, opnd1); - } - } - }, - Insn::Sub { left, right, .. } => { - *left = split_load_operand(asm, *left); - *right = split_shifted_immediate(asm, *right); + *opnd = split_load_operand(asm, *opnd); asm.push_insn(insn); }, Insn::Mul { left, right, .. } => { @@ -730,9 +690,234 @@ impl Assembler asm_local } + /// Split instructions using scratch registers. To maximize the use of the register pool for + /// VRegs, most splits should happen in [`Self::arm64_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so this + /// splits them and uses scratch registers for it. + fn arm64_scratch_split(mut self) -> Assembler { + /// If opnd is Opnd::Mem with a too large disp, make the disp smaller using lea. + fn split_large_disp(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + match opnd { + Opnd::Mem(Mem { num_bits, disp, .. }) if !mem_disp_fits_bits(disp) => { + asm.lea_into(scratch_opnd, opnd); + Opnd::mem(num_bits, scratch_opnd, 0) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem with MemBase::Stack, lower it to Opnd::Mem with MemBase::Reg, and split a large disp. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + let opnd = split_only_stack_membase(asm, opnd, scratch_opnd, stack_state); + split_large_disp(asm, opnd, scratch_opnd) + } + + /// split_stack_membase but without split_large_disp. This should be used only by lea. + fn split_only_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + if let Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp: opnd_disp, num_bits: opnd_num_bits }) = opnd { + let base = Opnd::Mem(stack_state.stack_membase_to_mem(stack_membase)); + let base = split_large_disp(asm, base, scratch_opnd); + asm.load_into(scratch_opnd, base); + Opnd::Mem(Mem { base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), disp: opnd_disp, num_bits: opnd_num_bits }) + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, lower it to scratch_opnd. You should use this when `opnd` is read by the instruction, not written. + fn split_memory_read(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + if let Opnd::Mem(_) = opnd { + let opnd = split_large_disp(asm, opnd, scratch_opnd); + let scratch_opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + asm.load_into(scratch_opnd, opnd); + scratch_opnd + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new_with_asm(&self); + let asm = &mut asm_local; + asm.accept_scratch_reg = true; + let iterator = &mut self.instruction_iterator(); + + while let Some((_, mut insn)) = iterator.next(asm) { + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } | + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND); + *right = split_memory_read(asm, *right, SCRATCH1_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Mul { left, right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND); + *right = split_memory_read(asm, *right, SCRATCH1_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + let reg_out = out.clone(); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + }; + + // If the next instruction is JoMul + if matches!(iterator.peek(), Some((_, Insn::JoMul(_)))) { + // Produce a register that is all zeros or all ones + // Based on the sign bit of the 64-bit mul result + asm.push_insn(Insn::RShift { out: SCRATCH0_OPND, opnd: reg_out, shift: Opnd::UImm(63) }); + } + } + Insn::LShift { opnd, out, .. } | + Insn::RShift { opnd, out, .. } => { + *opnd = split_memory_read(asm, *opnd, SCRATCH0_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Cmp { left, right } | + Insn::Test { left, right } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND); + *right = split_memory_read(asm, *right, SCRATCH1_OPND); + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + // For compile_exits, support splitting simple return values here + Insn::CRet(opnd) => { + match opnd { + Opnd::Reg(C_RET_REG) => {}, + _ => asm.load_into(C_RET_OPND, *opnd), + } + asm.cret(C_RET_OPND); + } + Insn::Lea { opnd, out } => { + *opnd = split_only_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_large_disp(asm, mem_out, SCRATCH1_OPND); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Load { opnd, out } | + Insn::LoadInto { opnd, dest: out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + *out = split_stack_membase(asm, *out, SCRATCH1_OPND, &stack_state); + + if let Opnd::Mem(_) = out { + // If NATIVE_STACK_PTR is used as a source for Store, it's handled as xzr, storeing zero. + // To save the content of NATIVE_STACK_PTR, we need to load it into another register first. + if *opnd == NATIVE_STACK_PTR { + asm.load_into(SCRATCH0_OPND, NATIVE_STACK_PTR); + *opnd = SCRATCH0_OPND; + } + asm.store(*out, *opnd); + } else { + asm.push_insn(insn); + } + } + &mut Insn::IncrCounter { mem, value } => { + // Convert Opnd::const_ptr into Opnd::Mem. + // It's split here to support IncrCounter in compile_exits. + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.lea_into(SCRATCH0_OPND, Opnd::mem(64, SCRATCH0_OPND, 0)); + + // Create a local loop to atomically increment a counter using SCRATCH1_OPND to check if it succeeded. + // Note that arm64_emit will peek at the next Cmp to set a status into SCRATCH1_OPND on IncrCounter. + let label = asm.new_label("incr_counter_loop"); + asm.write_label(label.clone()); + asm.incr_counter(SCRATCH0_OPND, value); + asm.cmp(SCRATCH1_OPND, 0.into()); + asm.jne(label); + } + Insn::Store { dest, .. } => { + *dest = split_stack_membase(asm, *dest, SCRATCH0_OPND, &stack_state); + asm.push_insn(insn); + } + Insn::Mov { dest, src } => { + *src = split_stack_membase(asm, *src, SCRATCH0_OPND, &stack_state); + *dest = split_large_disp(asm, *dest, SCRATCH1_OPND); + match dest { + Opnd::Reg(_) => asm.load_into(*dest, *src), + Opnd::Mem(_) => asm.store(*dest, *src), + _ => asm.push_insn(insn), + } + } + // Resolve ParallelMov that couldn't be handled without a scratch register. + Insn::ParallelMov { moves } => { + for (dst, src) in Self::resolve_parallel_moves(moves, Some(SCRATCH0_OPND)).unwrap() { + let src = split_stack_membase(asm, src, SCRATCH1_OPND, &stack_state); + let dst = split_large_disp(asm, dst, SCRATCH2_OPND); + match dst { + Opnd::Reg(_) => asm.load_into(dst, src), + Opnd::Mem(_) => asm.store(dst, src), + _ => asm.mov(dst, src), + } + } + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + /// Emit platform-specific machine code /// Returns a list of GC offsets. Can return failure to signal caller to retry. - fn arm64_emit(&mut self, cb: &mut CodeBlock) -> Result<Vec<u32>, EmitError> { + fn arm64_emit(&mut self, cb: &mut CodeBlock) -> Option<Vec<CodePtr>> { /// Determine how many instructions it will take to represent moving /// this value into a register. Note that the return value of this /// function must correspond to how many instructions are used to @@ -756,81 +941,78 @@ impl Assembler /// Emit a conditional jump instruction to a specific target. This is /// called when lowering any of the conditional jump instructions. fn emit_conditional_jump<const CONDITION: u8>(cb: &mut CodeBlock, target: Target) { + fn generate_branch<const CONDITION: u8>(cb: &mut CodeBlock, src_addr: i64, dst_addr: i64) { + let num_insns = if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { + // If the jump offset fits into the conditional jump as + // an immediate value and it's properly aligned, then we + // can use the b.cond instruction directly. We're safe + // to use as i32 here since we already checked that it + // fits. + let bytes = (dst_addr - src_addr) as i32; + bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); + + // Here we're going to return 1 because we've only + // written out 1 instruction. + 1 + } else if b_offset_fits_bits((dst_addr - (src_addr + 4)) / 4) { // + 4 for bcond + // If the jump offset fits into the unconditional jump as + // an immediate value, we can use inverse b.cond + b. + // + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instruction used for branching. + bcond(cb, Condition::inverse(CONDITION), 2.into()); + b(cb, InstructionOffset::from_bytes((dst_addr - (src_addr + 4)) as i32)); // + 4 for bcond + + // We've only written out 2 instructions. + 2 + } else { + // Otherwise, we need to load the address into a + // register and use the branch register instruction. + let load_insns: i32 = emit_load_size(dst_addr as u64).into(); + + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instructions used for branching. + bcond(cb, Condition::inverse(CONDITION), (load_insns + 2).into()); + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); + + // Here we'll return the number of instructions that it + // took to write out the destination address + 1 for the + // b.cond and 1 for the br. + load_insns + 2 + }; + + // We need to make sure we have at least 6 instructions for + // every kind of jump for invalidation purposes, so we're + // going to write out padding nop instructions here. + assert!(num_insns <= cb.conditional_jump_insns()); + (num_insns..cb.conditional_jump_insns()).for_each(|_| nop(cb)); + } + match target { - Target::CodePtr(dst_ptr) | Target::SideExitPtr(dst_ptr) => { + Target::CodePtr(dst_ptr) => { let dst_addr = dst_ptr.as_offset(); let src_addr = cb.get_write_ptr().as_offset(); - - let num_insns = if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { - // If the jump offset fits into the conditional jump as - // an immediate value and it's properly aligned, then we - // can use the b.cond instruction directly. We're safe - // to use as i32 here since we already checked that it - // fits. - let bytes = (dst_addr - src_addr) as i32; - bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); - - // Here we're going to return 1 because we've only - // written out 1 instruction. - 1 - } else if b_offset_fits_bits((dst_addr - (src_addr + 4)) / 4) { // + 4 for bcond - // If the jump offset fits into the unconditional jump as - // an immediate value, we can use inverse b.cond + b. - // - // We're going to write out the inverse condition so - // that if it doesn't match it will skip over the - // instruction used for branching. - bcond(cb, Condition::inverse(CONDITION), 2.into()); - b(cb, InstructionOffset::from_bytes((dst_addr - (src_addr + 4)) as i32)); // + 4 for bcond - - // We've only written out 2 instructions. - 2 - } else { - // Otherwise, we need to load the address into a - // register and use the branch register instruction. - let dst_addr = (dst_ptr.raw_ptr(cb) as usize).as_u64(); - let load_insns: i32 = emit_load_size(dst_addr).into(); - - // We're going to write out the inverse condition so - // that if it doesn't match it will skip over the - // instructions used for branching. - bcond(cb, Condition::inverse(CONDITION), (load_insns + 2).into()); - emit_load_value(cb, Assembler::SCRATCH0, dst_addr); - br(cb, Assembler::SCRATCH0); - - // Here we'll return the number of instructions that it - // took to write out the destination address + 1 for the - // b.cond and 1 for the br. - load_insns + 2 - }; - - if let Target::CodePtr(_) = target { - // We need to make sure we have at least 6 instructions for - // every kind of jump for invalidation purposes, so we're - // going to write out padding nop instructions here. - assert!(num_insns <= cb.conditional_jump_insns()); - for _ in num_insns..cb.conditional_jump_insns() { nop(cb); } - } + generate_branch::<CONDITION>(cb, src_addr, dst_addr); }, Target::Label(label_idx) => { - // Here we're going to save enough space for ourselves and - // then come back and write the instruction once we know the - // offset. We're going to assume we can fit into a single - // b.cond instruction. It will panic otherwise. - cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { - let bytes: i32 = (dst_addr - (src_addr - 4)).try_into().unwrap(); - bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); + // We save `cb.conditional_jump_insns` number of bytes since we may use up to that amount + // `generate_branch` will pad the emitted branch instructions with `nop`s for each unused byte. + cb.label_ref(label_idx, (cb.conditional_jump_insns() * 4) as usize, |cb, src_addr, dst_addr| { + generate_branch::<CONDITION>(cb, src_addr - (cb.conditional_jump_insns() * 4) as i64, dst_addr); }); }, Target::SideExit { .. } => { - unreachable!("Target::SideExit should have been compiled by compile_side_exits") + unreachable!("Target::SideExit should have been compiled by compile_exits") }, }; } /// Emit a CBZ or CBNZ which branches when a register is zero or non-zero fn emit_cmp_zero_jump(cb: &mut CodeBlock, reg: A64Opnd, branch_if_zero: bool, target: Target) { - if let Target::SideExitPtr(dst_ptr) = target { + if let Target::CodePtr(dst_ptr) = target { let dst_addr = dst_ptr.as_offset(); let src_addr = cb.get_write_ptr().as_offset(); @@ -858,15 +1040,52 @@ impl Assembler } else { cbz(cb, reg, InstructionOffset::from_insns(load_insns + 2)); } - emit_load_value(cb, Assembler::SCRATCH0, dst_addr); - br(cb, Assembler::SCRATCH0); - + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr); + br(cb, Assembler::EMIT_OPND); } } else { unreachable!("We should only generate Joz/Jonz with side-exit targets"); } } + /// Do the address calculation of `out_reg = base_reg + disp` + fn load_effective_address(cb: &mut CodeBlock, out: A64Opnd, base_reg_no: u8, disp: i32) { + let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + assert_ne!(base_reg_no, out_reg_no, "large displacement need a scratch register"); + + if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { + // Use ADD/SUB if the displacement fits + add(cb, out, base_reg, A64Opnd::new_imm(disp.into())); + } else { + // Use add_extended() to interpret reg_no=31 as sp + // since the base register is never the zero register. + // Careful! Only the first two operands can refer to sp. + emit_load_value(cb, out, disp as u64); + add_extended(cb, out, base_reg, out); + }; + } + + /// Load a VALUE to a register and remember it for GC marking and reference updating + fn emit_load_gc_value(cb: &mut CodeBlock, gc_offsets: &mut Vec<CodePtr>, dest: A64Opnd, value: VALUE) { + // We dont need to check if it's a special const + // here because we only allow these operands to hit + // this point if they're not a special const. + assert!(!value.special_const_p()); + + // This assumes only load instructions can contain + // references to GC'd Value operands. If the value + // being loaded is a heap object, we'll report that + // back out to the gc_offsets list. + ldr_literal(cb, dest, 2.into()); + b(cb, InstructionOffset::from_bytes(4 + (SIZEOF_VALUE as i32))); + cb.write_bytes(&value.as_u64().to_le_bytes()); + + let ptr_offset = cb.get_write_ptr().sub_bytes(SIZEOF_VALUE); + gc_offsets.push(ptr_offset); + } + /// Emit a push instruction for the given operand by adding to the stack /// pointer and then storing the given value. fn emit_push(cb: &mut CodeBlock, opnd: A64Opnd) { @@ -879,22 +1098,23 @@ impl Assembler ldr_post(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); } - // dbg!(&self.insns); - // List of GC offsets - let mut gc_offsets: Vec<u32> = Vec::new(); + let mut gc_offsets: Vec<CodePtr> = Vec::new(); // Buffered list of PosMarker callbacks to fire if codegen is successful let mut pos_markers: Vec<(usize, CodePtr)> = vec![]; + // The write_pos for the last Insn::PatchPoint, if any + let mut last_patch_pos: Option<usize> = None; + + // Install a panic hook to dump Assembler with insn_idx on dev builds + let (_hook, mut hook_insn_idx) = AssemblerPanicHook::new(self, 0); + // For each instruction - //let start_write_pos = cb.get_write_pos(); let mut insn_idx: usize = 0; while let Some(insn) = self.insns.get(insn_idx) { - //let src_ptr = cb.get_write_ptr(); - let had_dropped_bytes = cb.has_dropped_bytes(); - //let old_label_state = cb.get_label_state(); - let mut insn_gc_offsets: Vec<u32> = Vec::new(); + // Update insn_idx that is shown on panic + hook_insn_idx.as_mut().map(|idx| idx.lock().map(|mut idx| *idx = insn_idx).unwrap()); match insn { Insn::Comment(text) => { @@ -922,44 +1142,98 @@ impl Assembler cb.write_byte(0); } }, - Insn::FrameSetup => { + &Insn::FrameSetup { preserved, mut slot_count } => { + const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); } + // Preserve X29 and set up frame record stp_pre(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, -16)); - - // X29 (frame_pointer) = SP mov(cb, X29, C_SP_REG); - }, - Insn::FrameTeardown => { + + for regs in preserved.chunks(2) { + // For the body, store pairs and move SP + if let [reg0, reg1] = regs { + stp_pre(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, C_SP_REG, -16)); + } else if let [reg] = regs { + // For overhang, store but don't move SP. Combine movement with + // movement for slots below. + stur(cb, reg.into(), A64Opnd::new_mem(64, C_SP_REG, -8)); + slot_count += 1; + } else { + unreachable!("chunks(2)"); + } + } + // Align slot_count + if slot_count % 2 == 1 { + slot_count += 1 + } + if slot_count > 0 { + let slot_offset = (slot_count * SIZEOF_VALUE) as u64; + // Bail when asked to reserve too many slots in one instruction. + ShiftedImmediate::try_from(slot_offset).ok()?; + sub(cb, C_SP_REG, C_SP_REG, A64Opnd::new_uimm(slot_offset)); + } + } + Insn::FrameTeardown { preserved } => { + // Restore preserved registers below frame pointer. + let mut base_offset = 0; + for regs in preserved.chunks(2) { + if let [reg0, reg1] = regs { + base_offset -= 16; + ldp(cb, reg1.into(), reg0.into(), A64Opnd::new_mem(128, X29, base_offset)); + } else if let [reg] = regs { + ldur(cb, reg.into(), A64Opnd::new_mem(64, X29, base_offset - 8)); + } else { + unreachable!("chunks(2)"); + } + } + // SP = X29 (frame pointer) mov(cb, C_SP_REG, X29); - ldp_post(cb, X29, X30, A64Opnd::new_mem(128, C_SP_REG, 16)); - }, + } Insn::Add { left, right, out } => { - adds(cb, out.into(), left.into(), right.into()); + // Usually, we issue ADDS, so you could branch on overflow, but ADDS with + // out=31 refers to out=XZR, which discards the sum. So, instead of ADDS + // (aliased to CMN in this case) we issue ADD instead which writes the sum + // to the stack pointer; we assume you got x31 from NATIVE_STACK_POINTER. + let out: A64Opnd = out.into(); + if let A64Opnd::Reg(A64Reg { reg_no: 31, .. }) = out { + add(cb, out, left.into(), right.into()); + } else { + adds(cb, out, left.into(), right.into()); + } }, Insn::Sub { left, right, out } => { - subs(cb, out.into(), left.into(), right.into()); + // Usually, we issue SUBS, so you could branch on overflow, but SUBS with + // out=31 refers to out=XZR, which discards the result. So, instead of SUBS + // (aliased to CMP in this case) we issue SUB instead which writes the diff + // to the stack pointer; we assume you got x31 from NATIVE_STACK_POINTER. + let out: A64Opnd = out.into(); + if let A64Opnd::Reg(A64Reg { reg_no: 31, .. }) = out { + sub(cb, out, left.into(), right.into()); + } else { + subs(cb, out, left.into(), right.into()); + } }, Insn::Mul { left, right, out } => { - // If the next instruction is jo (jump on overflow) + // If the next instruction is JoMul with RShift created by arm64_scratch_split match (self.insns.get(insn_idx + 1), self.insns.get(insn_idx + 2)) { - (Some(Insn::JoMul(_)), _) | - (Some(Insn::PosMarker(_)), Some(Insn::JoMul(_))) => { + (Some(Insn::RShift { out: out_sign, opnd: out_opnd, shift: out_shift }), Some(Insn::JoMul(_))) => { // Compute the high 64 bits - smulh(cb, Self::SCRATCH0, left.into(), right.into()); + smulh(cb, Self::EMIT_OPND, left.into(), right.into()); // Compute the low 64 bits // This may clobber one of the input registers, // so we do it after smulh mul(cb, out.into(), left.into(), right.into()); - // Produce a register that is all zeros or all ones - // Based on the sign bit of the 64-bit mul result - asr(cb, Self::SCRATCH1, out.into(), A64Opnd::UImm(63)); + // Insert the shift instruction created by arm64_scratch_split + // to prepare the register that has the sign bit of the high 64 bits after mul. + asr(cb, out_sign.into(), out_opnd.into(), out_shift.into()); + insn_idx += 1; // skip the next Insn::RShift // If the high 64-bits are not all zeros or all ones, // matching the sign bit, then we have an overflow - cmp(cb, Self::SCRATCH0, Self::SCRATCH1); + cmp(cb, Self::EMIT_OPND, out_sign.into()); // Insn::JoMul will emit_conditional_jump::<{Condition::NE}> } _ => { @@ -989,14 +1263,55 @@ impl Assembler lsl(cb, out.into(), opnd.into(), shift.into()); }, Insn::Store { dest, src } => { + // Split src into EMIT0_OPND if necessary + let src_reg: A64Reg = match src { + Opnd::Reg(reg) => *reg, + // Use zero register when possible + Opnd::UImm(0) | Opnd::Imm(0) => XZR_REG, + // Immediates + &Opnd::Imm(imm) => { + emit_load_value(cb, Self::EMIT_OPND, imm as u64); + Self::EMIT_REG + } + &Opnd::UImm(imm) => { + emit_load_value(cb, Self::EMIT_OPND, imm); + Self::EMIT_REG + } + &Opnd::Value(value) => { + emit_load_gc_value(cb, &mut gc_offsets, Self::EMIT_OPND, value); + Self::EMIT_REG + } + src_mem @ &Opnd::Mem(Mem { num_bits: src_num_bits, base: MemBase::Reg(src_base_reg_no), disp: src_disp }) => { + // For mem-to-mem store, load the source into EMIT0_OPND + let src_mem = if mem_disp_fits_bits(src_disp) { + src_mem.into() + } else { + // Split the load address into EMIT0_OPND first if necessary + load_effective_address(cb, Self::EMIT_OPND, src_base_reg_no, src_disp); + A64Opnd::new_mem(dest.rm_num_bits(), Self::EMIT_OPND, 0) + }; + let dst = A64Opnd::Reg(Self::EMIT_REG.with_num_bits(src_num_bits)); + match src_num_bits { + 64 | 32 => ldur(cb, dst, src_mem), + 16 => ldurh(cb, dst, src_mem), + 8 => ldurb(cb, dst, src_mem), + num_bits => panic!("unexpected num_bits: {num_bits}") + }; + Self::EMIT_REG + } + src @ (Opnd::Mem(_) | Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during arm64_emit: {src:?}") + }; + let src = A64Opnd::Reg(src_reg); + // This order may be surprising but it is correct. The way // the Arm64 assembler works, the register that is going to // be stored is first and the address is second. However in // our IR we have the address first and the register second. match dest.rm_num_bits() { - 64 | 32 => stur(cb, src.into(), dest.into()), - 16 => sturh(cb, src.into(), dest.into()), - num_bits => panic!("unexpected dest num_bits: {} (src: {:#?}, dest: {:#?})", num_bits, src, dest), + 64 | 32 => stur(cb, src, dest.into()), + 16 => sturh(cb, src, dest.into()), + 8 => sturb(cb, src, dest.into()), + num_bits => panic!("unexpected dest num_bits: {num_bits} (src: {src:?}, dest: {dest:?})"), } }, Insn::Load { opnd, out } | @@ -1016,25 +1331,11 @@ impl Assembler 64 | 32 => ldur(cb, out.into(), opnd.into()), 16 => ldurh(cb, out.into(), opnd.into()), 8 => ldurb(cb, out.into(), opnd.into()), - num_bits => panic!("unexpected num_bits: {}", num_bits) + num_bits => panic!("unexpected num_bits: {num_bits}"), }; }, Opnd::Value(value) => { - // We dont need to check if it's a special const - // here because we only allow these operands to hit - // this point if they're not a special const. - assert!(!value.special_const_p()); - - // This assumes only load instructions can contain - // references to GC'd Value operands. If the value - // being loaded is a heap object, we'll report that - // back out to the gc_offsets list. - ldr_literal(cb, out.into(), 2.into()); - b(cb, InstructionOffset::from_bytes(4 + (SIZEOF_VALUE as i32))); - cb.write_bytes(&value.as_u64().to_le_bytes()); - - let ptr_offset: u32 = (cb.get_write_pos() as u32) - (SIZEOF_VALUE as u32); - insn_gc_offsets.push(ptr_offset); + emit_load_gc_value(cb, &mut gc_offsets, out.into(), value); }, Opnd::None => { unreachable!("Attempted to load from None operand"); @@ -1069,30 +1370,39 @@ impl Assembler } }, Insn::Lea { opnd, out } => { - let opnd: A64Opnd = opnd.into(); - - match opnd { - A64Opnd::Mem(mem) => { - add( - cb, - out.into(), - A64Opnd::Reg(A64Reg { reg_no: mem.base_reg_no, num_bits: 64 }), - A64Opnd::new_imm(mem.disp.into()) - ); - }, - _ => { - panic!("Op::Lea only accepts Opnd::Mem operands."); - } + let &Opnd::Mem(Mem { num_bits: _, base: MemBase::Reg(base_reg_no), disp }) = opnd else { + panic!("Unexpected Insn::Lea operand in arm64_emit: {opnd:?}"); }; - }, + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + + let out = A64Opnd::from(out); + let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); + if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { + // Use ADD/SUB if the displacement fits + add(cb, out, base_reg, A64Opnd::new_imm(disp.into())); + } else { + // Use a scratch reg for `out += displacement` + let disp_reg = if out_reg_no == base_reg_no { + Self::EMIT_OPND + } else { + out + }; + // Use add_extended() to interpret reg_no=31 as sp + // since the base register is never the zero register. + // Careful! Only the first two operands can refer to sp. + emit_load_value(cb, disp_reg, disp as u64); + add_extended(cb, out, base_reg, disp_reg); + } + } Insn::LeaJumpTarget { out, target, .. } => { if let Target::Label(label_idx) = target { // Set output to the raw address of the label cb.label_ref(*label_idx, 4, |cb, end_addr, dst_addr| { - adr(cb, Self::SCRATCH0, A64Opnd::new_imm(dst_addr - (end_addr - 4))); + adr(cb, Self::EMIT_OPND, A64Opnd::new_imm(dst_addr - (end_addr - 4))); }); - mov(cb, out.into(), Self::SCRATCH0); + mov(cb, out.into(), Self::EMIT_OPND); } else { // Set output to the jump target's raw address let target_code = target.unwrap_code_ptr(); @@ -1117,32 +1427,40 @@ impl Assembler } // Push the flags/state register - mrs(cb, Self::SCRATCH0, SystemRegister::NZCV); - emit_push(cb, Self::SCRATCH0); + mrs(cb, Self::EMIT_OPND, SystemRegister::NZCV); + emit_push(cb, Self::EMIT_OPND); }, Insn::CPopAll => { let regs = Assembler::get_caller_save_regs(); // Pop the state/flags register - msr(cb, SystemRegister::NZCV, Self::SCRATCH0); - emit_pop(cb, Self::SCRATCH0); + msr(cb, SystemRegister::NZCV, Self::EMIT_OPND); + emit_pop(cb, Self::EMIT_OPND); for reg in regs.into_iter().rev() { emit_pop(cb, A64Opnd::Reg(reg)); } }, Insn::CCall { fptr, .. } => { - // The offset to the call target in bytes - let src_addr = cb.get_write_ptr().raw_ptr(cb) as i64; - let dst_addr = *fptr as i64; - - // Use BL if the offset is short enough to encode as an immediate. - // Otherwise, use BLR with a register. - if b_offset_fits_bits((dst_addr - src_addr) / 4) { - bl(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); - } else { - emit_load_value(cb, Self::SCRATCH0, dst_addr as u64); - blr(cb, Self::SCRATCH0); + match fptr { + Opnd::UImm(fptr) => { + // The offset to the call target in bytes + let src_addr = cb.get_write_ptr().raw_ptr(cb) as i64; + let dst_addr = *fptr as i64; + + // Use BL if the offset is short enough to encode as an immediate. + // Otherwise, use BLR with a register. + if b_offset_fits_bits((dst_addr - src_addr) / 4) { + bl(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); + } else { + emit_load_value(cb, Self::EMIT_OPND, dst_addr as u64); + blr(cb, Self::EMIT_OPND); + } + } + Opnd::Reg(_) => { + blr(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") } }, Insn::CRet { .. } => { @@ -1162,9 +1480,6 @@ impl Assembler Target::CodePtr(dst_ptr) => { emit_jmp_ptr(cb, dst_ptr, true); }, - Target::SideExitPtr(dst_ptr) => { - emit_jmp_ptr(cb, dst_ptr, false); - }, Target::Label(label_idx) => { // Here we're going to save enough space for // ourselves and then come back and write the @@ -1177,7 +1492,7 @@ impl Assembler }); }, Target::SideExit { .. } => { - unreachable!("Target::SideExit should have been compiled by compile_side_exits") + unreachable!("Target::SideExit should have been compiled by compile_exits") }, }; }, @@ -1211,25 +1526,35 @@ impl Assembler Insn::Jonz(opnd, target) => { emit_cmp_zero_jump(cb, opnd.into(), false, target.clone()); }, - Insn::IncrCounter { mem: _, value: _ } => { - /* - let label = cb.new_label("incr_counter_loop".to_string()); - cb.write_label(label); + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in arm64_scratch_split"), + Insn::PadPatchPoint => { + // If patch points are too close to each other or the end of the block, fill nop instructions + if let Some(last_patch_pos) = last_patch_pos { + while cb.get_write_pos().saturating_sub(last_patch_pos) < cb.jmp_ptr_bytes() && !cb.has_dropped_bytes() { + nop(cb); + } + } + last_patch_pos = Some(cb.get_write_pos()); + }, + Insn::IncrCounter { mem, value } => { + // Get the status register allocated by arm64_scratch_split + let Some(Insn::Cmp { + left: status_reg @ Opnd::Reg(_), + right: Opnd::UImm(_) | Opnd::Imm(_), + }) = self.insns.get(insn_idx + 1) else { + panic!("arm64_scratch_split should add Cmp after IncrCounter: {:?}", self.insns.get(insn_idx + 1)); + }; - ldaxr(cb, Self::SCRATCH0, mem.into()); - add(cb, Self::SCRATCH0, Self::SCRATCH0, value.into()); + // Attempt to increment a counter + ldaxr(cb, Self::EMIT_OPND, mem.into()); + add(cb, Self::EMIT_OPND, Self::EMIT_OPND, value.into()); // The status register that gets used to track whether or // not the store was successful must be 32 bytes. Since we - // store the SCRATCH registers as their 64-bit versions, we + // store the EMIT registers as their 64-bit versions, we // need to rewrap it here. - let status = A64Opnd::Reg(Self::SCRATCH1.unwrap_reg().with_num_bits(32)); - stlxr(cb, status, Self::SCRATCH0, mem.into()); - - cmp(cb, Self::SCRATCH1, A64Opnd::new_uimm(0)); - emit_conditional_jump::<{Condition::NE}>(cb, Target::Label(label)); - */ - unimplemented!("labels are not supported yet"); + let status = A64Opnd::Reg(status_reg.unwrap_reg().with_num_bits(32)); + stlxr(cb, status, Self::EMIT_OPND, mem.into()); }, Insn::Breakpoint => { brk(cb, A64Opnd::None); @@ -1255,30 +1580,14 @@ impl Assembler csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GE); } Insn::LiveReg { .. } => (), // just a reg alloc signal, no code - Insn::PadInvalPatch => { - unimplemented!("we haven't needed padding in ZJIT yet"); - } }; - // On failure, jump to the next page and retry the current insn - if !had_dropped_bytes && cb.has_dropped_bytes() { - // Reset cb states before retrying the current Insn - //cb.set_label_state(old_label_state); - - // We don't want label references to cross page boundaries. Signal caller for - // retry. - if !self.label_names.is_empty() { - return Err(EmitError::RetryOnNextPage); - } - } else { - insn_idx += 1; - gc_offsets.append(&mut insn_gc_offsets); - } + insn_idx += 1; } // Error if we couldn't write out everything if cb.has_dropped_bytes() { - Err(EmitError::OutOfMemory) + None } else { // No bytes dropped, so the pos markers point to valid code for (insn_idx, pos) in pos_markers { @@ -1289,14 +1598,30 @@ impl Assembler } } - Ok(gc_offsets) + Some(gc_offsets) } } /// Optimize and compile the stored instructions - pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Option<(CodePtr, Vec<u32>)> { + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_reg = !self.accept_scratch_reg; + asm_dump!(self, init); + let asm = self.arm64_split(); - let mut asm = asm.alloc_regs(regs); + asm_dump!(asm, split); + + let mut asm = asm.alloc_regs(regs)?; + asm_dump!(asm, alloc_regs); + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + asm.compile_exits(); + asm_dump!(asm, compile_exits); + + if use_scratch_reg { + asm = asm.arm64_scratch_split(); + asm_dump!(asm, scratch_split); + } // Create label instances in the code block for (idx, name) in asm.label_names.iter().enumerate() { @@ -1305,48 +1630,188 @@ impl Assembler } let start_ptr = cb.get_write_ptr(); - /* - let starting_label_state = cb.get_label_state(); - let emit_result = match asm.arm64_emit(cb, &mut ocb) { - Err(EmitError::RetryOnNextPage) => { - // we want to lower jumps to labels to b.cond instructions, which have a 1 MiB - // range limit. We can easily exceed the limit in case the jump straddles two pages. - // In this case, we retry with a fresh page once. - cb.set_label_state(starting_label_state); - if cb.next_page(start_ptr, emit_jmp_ptr_with_invalidation) { - asm.arm64_emit(cb, &mut ocb) - } else { - Err(EmitError::OutOfMemory) - } - } - result => result - }; - */ - let emit_result = asm.arm64_emit(cb); + let gc_offsets = asm.arm64_emit(cb); - if let (Ok(gc_offsets), false) = (emit_result, cb.has_dropped_bytes()) { + if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { cb.link_labels(); // Invalidate icache for newly written out region so we don't run stale code. - unsafe { rb_zjit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; - Some((start_ptr, gc_offsets)) + Ok((start_ptr, gc_offsets)) } else { cb.clear_labels(); + Err(CompileError::OutOfMemory) + } + } +} - None +/// LIR Instructions that are lowered to an instruction that have 2 input registers and an output +/// register can look to merge with a succeeding `Insn::Mov`. +/// For example: +/// +/// Add out, a, b +/// Mov c, out +/// +/// Can become: +/// +/// Add c, a, b +/// +/// If a, b, and c are all registers. +fn merge_three_reg_mov( + live_ranges: &[LiveRange], + iterator: &mut InsnIter, + asm: &mut Assembler, + left: &Opnd, + right: &Opnd, + out: &mut Opnd, +) { + if let (Opnd::Reg(_) | Opnd::VReg{..}, + Opnd::Reg(_) | Opnd::VReg{..}, + Some((mov_idx, Insn::Mov { dest, src }))) + = (left, right, iterator.peek()) { + if out == src && live_ranges[out.vreg_idx()].end() == *mov_idx && matches!(*dest, Opnd::Reg(_) | Opnd::VReg{..}) { + *out = *dest; + iterator.next(asm); // Pop merged Insn::Mov } } } -/* #[cfg(test)] mod tests { + #[cfg(feature = "disasm")] + use crate::disasms_with; + use crate::{assert_disasm_snapshot, hexdumps}; + use super::*; - use crate::disasm::*; + use insta::assert_snapshot; + + static TEMP_REGS: [Reg; 5] = [X1_REG, X9_REG, X10_REG, X14_REG, X15_REG]; fn setup_asm() -> (Assembler, CodeBlock) { - (Assembler::new(0), CodeBlock::new_dummy(1024)) + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + (Assembler::new(), CodeBlock::new_dummy()) + } + + #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.parallel_mov(vec![(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)), (C_ARG_OPNDS[1], Opnd::mem(64, SP, -8))]); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.je(label); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @r" + bb0: + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, x19, x21, x20 + v0 = Add x19, 0x40 + Store [x21 + 0x10], v0 + Joz Exit(Interrupt), v0 + ParallelMov x0 <- w0, x1 <- [x21 - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[x20 + 0x10], VReg32(v1) + Je bb0 + CRet v0 + FrameTeardown x19, x21, x20 + "); + } + + #[test] + fn test_mul_with_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.mul(Opnd::Reg(TEMP_REGS[1]), 3.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #3 + 0x4: mul x0, x9, x0 + 0x8: mov x1, x0 + "); + assert_snapshot!(cb.hexdump(), @"600080d2207d009be10300aa"); + } + + #[test] + fn sp_movements_are_single_instruction() { + let (mut asm, mut cb) = setup_asm(); + + let sp = Opnd::Reg(XZR_REG); + let new_sp = asm.add(sp, 0x20.into()); + asm.mov(sp, new_sp); + let new_sp = asm.sub(sp, 0x20.into()); + asm.mov(sp, new_sp); + + asm.compile_with_num_regs(&mut cb, 2); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add sp, sp, #0x20 + 0x4: sub sp, sp, #0x20 + "); + assert_snapshot!(cb.hexdump(), @"ff830091ff8300d1"); + } + + #[test] + fn add_into() { + let (mut asm, mut cb) = setup_asm(); + + let sp = Opnd::Reg(XZR_REG); + asm.add_into(sp, 8.into()); + asm.add_into(Opnd::Reg(X20_REG), 0x20.into()); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add sp, sp, #8 + 0x4: adds x20, x20, #0x20 + "); + assert_snapshot!(cb.hexdump(), @"ff230091948200b1"); + } + + #[test] + fn sub_imm_reg() { + let (mut asm, mut cb) = setup_asm(); + + let difference = asm.sub(0x8.into(), Opnd::Reg(X5_REG)); + asm.load_into(Opnd::Reg(X1_REG), difference); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #8 + 0x4: subs x0, x0, x5 + 0x8: mov x1, x0 + "); + assert_snapshot!(cb.hexdump(), @"000180d2000005ebe10300aa"); + } + + #[test] + fn no_dead_mov_from_vreg() { + let (mut asm, mut cb) = setup_asm(); + + let ret_val = asm.load(Opnd::mem(64, C_RET_OPND, 0)); + asm.cret(ret_val); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [x0] + 0x4: ret + "); + assert_snapshot!(cb.hexdump(), @"000040f8c0035fd6"); } #[test] @@ -1355,10 +1820,14 @@ mod tests { let opnd = asm.add(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); - asm.compile_with_regs(&mut cb, None, vec![X3_REG]); + asm.compile_with_regs(&mut cb, vec![X3_REG]).unwrap(); // Assert that only 2 instructions were written. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: adds x3, x0, x1 + 0x4: stur x3, [x2] + "); + assert_snapshot!(cb.hexdump(), @"030001ab430000f8"); } #[test] @@ -1370,7 +1839,13 @@ mod tests { // Testing that we pad the string to the nearest 4-byte boundary to make // it easier to jump over. - assert_eq!(16, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldnp d8, d25, [x10, #-0x140] + 0x4: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x8: .byte 0x6f, 0x72, 0x6c, 0x64 + 0xc: .byte 0x21, 0x00, 0x00, 0x00 + "); + assert_snapshot!(cb.hexdump(), @"48656c6c6f2c20776f726c6421000000"); } #[test] @@ -1379,6 +1854,20 @@ mod tests { asm.cpush_all(); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: str x1, [sp, #-0x10]! + 0x4: str x9, [sp, #-0x10]! + 0x8: str x10, [sp, #-0x10]! + 0xc: str x11, [sp, #-0x10]! + 0x10: str x12, [sp, #-0x10]! + 0x14: str x13, [sp, #-0x10]! + 0x18: str x14, [sp, #-0x10]! + 0x1c: str x15, [sp, #-0x10]! + 0x20: mrs x16, nzcv + 0x24: str x16, [sp, #-0x10]! + "); + assert_snapshot!(cb.hexdump(), @"e10f1ff8e90f1ff8ea0f1ff8eb0f1ff8ec0f1ff8ed0f1ff8ee0f1ff8ef0f1ff810423bd5f00f1ff8"); } #[test] @@ -1387,15 +1876,109 @@ mod tests { asm.cpop_all(); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: msr nzcv, x16 + 0x4: ldr x16, [sp], #0x10 + 0x8: ldr x15, [sp], #0x10 + 0xc: ldr x14, [sp], #0x10 + 0x10: ldr x13, [sp], #0x10 + 0x14: ldr x12, [sp], #0x10 + 0x18: ldr x11, [sp], #0x10 + 0x1c: ldr x10, [sp], #0x10 + 0x20: ldr x9, [sp], #0x10 + 0x24: ldr x1, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"10421bd5f00741f8ef0741f8ee0741f8ed0741f8ec0741f8eb0741f8ea0741f8e90741f8e10741f8"); } #[test] fn test_emit_frame() { let (mut asm, mut cb) = setup_asm(); - asm.frame_setup(); - asm.frame_teardown(); + asm.frame_setup(&[]); + asm.frame_teardown(&[]); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: mov sp, x29 + 0xc: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"fd7bbfa9fd030091bf030091fd7bc1a8"); + } + + #[test] + fn frame_setup_and_teardown() { + const THREE_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG)]; + // Test 3 preserved regs (odd), odd slot_count + let cb1 = { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 3; + asm.frame_setup(THREE_REGS); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + // Test 3 preserved regs (odd), even slot_count + let cb2 = { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 4; + asm.frame_setup(THREE_REGS); + asm.frame_teardown(THREE_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + // Test 4 preserved regs (even), odd slot_count + let cb3 = { + static FOUR_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG), Opnd::Reg(X22_REG)]; + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 3; + asm.frame_setup(FOUR_REGS); + asm.frame_teardown(FOUR_REGS); + asm.compile_with_num_regs(&mut cb, 0); + cb + }; + + assert_disasm_snapshot!(disasms_with!("\n", cb1, cb2, cb3), @r" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x30 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stp x22, x21, [sp, #-0x10]! + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldp x22, x21, [x29, #-0x20] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(hexdumps!(cb1, cb2, cb3), @r" + fd7bbfa9fd030091f44fbfa9f5831ff8ff8300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f5831ff8ffc300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f657bfa9ff8300d1b44f7fa9b6577ea9bf030091fd7bc1a8 + "); } #[test] @@ -1406,6 +1989,16 @@ mod tests { asm.je(Target::CodePtr(target)); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: b.eq #0x50 + 0x4: nop + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"800200541f2003d51f2003d51f2003d51f2003d51f2003d5"); } #[test] @@ -1417,6 +2010,160 @@ mod tests { asm.je(Target::CodePtr(target)); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: b.ne #8 + 0x4: b #0x200000 + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"41000054ffff07141f2003d51f2003d51f2003d51f2003d5"); + } + + #[test] + fn test_emit_lea() { + let (mut asm, mut cb) = setup_asm(); + + // Test values that exercise various types of immediates. + // - 9 bit displacement for Load/Store + // - 12 bit ADD/SUB shifted immediate + // - 16 bit MOV family shifted immediates + // - bit mask immediates + for displacement in [i32::MAX, 0x10008, 0x1800, 0x208, -0x208, -0x1800, -0x10008, i32::MIN] { + let mem = Opnd::mem(64, NATIVE_STACK_PTR, displacement); + asm.lea_into(Opnd::Reg(X0_REG), mem); + } + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: orr x0, xzr, #0x7fffffff + 0x4: add x0, sp, x0 + 0x8: mov x0, #8 + 0xc: movk x0, #1, lsl #16 + 0x10: add x0, sp, x0 + 0x14: mov x0, #0x1800 + 0x18: add x0, sp, x0 + 0x1c: add x0, sp, #0x208 + 0x20: sub x0, sp, #0x208 + 0x24: mov x0, #-0x1800 + 0x28: add x0, sp, x0 + 0x2c: mov x0, #0xfff8 + 0x30: movk x0, #0xfffe, lsl #16 + 0x34: movk x0, #0xffff, lsl #32 + 0x38: movk x0, #0xffff, lsl #48 + 0x3c: add x0, sp, x0 + 0x40: orr x0, xzr, #0xffffffff80000000 + 0x44: add x0, sp, x0 + "); + assert_snapshot!(cb.hexdump(), @"e07b40b2e063208b000180d22000a0f2e063208b000083d2e063208be0230891e02308d1e0ff8292e063208b00ff9fd2c0ffbff2e0ffdff2e0fffff2e063208be08361b2e063208b"); + } + + #[test] + fn test_load_larg_disp_mem() { + let (mut asm, mut cb) = setup_asm(); + + let extended_ivars = asm.load(Opnd::mem(64, NATIVE_STACK_PTR, 0)); + let result = asm.load(Opnd::mem(VALUE_BITS, extended_ivars, 1000 * SIZEOF_VALUE_I32)); + asm.store(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0), result); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur x0, [sp] + 0x4: mov x16, #0x1f40 + 0x8: add x0, x0, x16, uxtx + 0xc: ldur x0, [x0] + 0x10: stur x0, [sp] + "); + assert_snapshot!(cb.hexdump(), @"e00340f810e883d20060308b000040f8e00300f8"); + } + + #[test] + fn test_store() { + let (mut asm, mut cb) = setup_asm(); + + // Large memory offsets in combinations of destination and source + let large_mem = Opnd::mem(64, NATIVE_STACK_PTR, -0x305); + let small_mem = Opnd::mem(64, C_RET_OPND, 0); + asm.store(small_mem, large_mem); + asm.store(large_mem, small_mem); + asm.store(large_mem, large_mem); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: sub x16, sp, #0x305 + 0x4: ldur x16, [x16] + 0x8: stur x16, [x0] + 0xc: sub x15, sp, #0x305 + 0x10: ldur x16, [x0] + 0x14: stur x16, [x15] + 0x18: sub x15, sp, #0x305 + 0x1c: sub x16, sp, #0x305 + 0x20: ldur x16, [x16] + 0x24: stur x16, [x15] + "); + assert_snapshot!(cb.hexdump(), @"f0170cd1100240f8100000f8ef170cd1100040f8f00100f8ef170cd1f0170cd1100240f8f00100f8"); + } + + #[test] + fn test_store_value_without_split() { + let (mut asm, mut cb) = setup_asm(); + + let imitation_heap_value = VALUE(0x1000); + assert!(imitation_heap_value.heap_object_p()); + asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into()); + + // Side exit code are compiled without the split pass, so we directly call emit here to + // emulate that scenario. + let gc_offsets = asm.arm64_emit(&mut cb).unwrap(); + assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldr x16, #8 + 0x4: b #0x10 + 0x8: .byte 0x00, 0x10, 0x00, 0x00 + 0xc: .byte 0x00, 0x00, 0x00, 0x00 + 0x10: stur x16, [x21] + "); + assert_snapshot!(cb.hexdump(), @"50000058030000140010000000000000b00200f8"); + } + + #[test] + fn test_store_with_valid_scratch_reg() { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + let mut cb = CodeBlock::new_dummy(); + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #0x3902 + 0x4: movk x16, #8, lsl #16 + 0x8: stur x16, [x15] + "); + assert_snapshot!(cb.hexdump(), @"502087d21001a0f2f00100f8"); + } + + #[test] + #[should_panic] + fn test_store_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); + let (mut asm, mut cb) = setup_asm(); + // This would put the source into scratch_reg, messing up the destination + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + } + + #[test] + #[should_panic] + fn test_load_into_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); + let (mut asm, mut cb) = setup_asm(); + // This would put the source into scratch_reg, messing up the destination + asm.load_into(scratch_reg, 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); } #[test] @@ -1424,13 +2171,23 @@ mod tests { let (mut asm, mut cb) = setup_asm(); let label = asm.new_label("label"); - let opnd = asm.lea_jump_target(label); + let opnd = asm.lea_jump_target(label.clone()); asm.write_label(label); asm.bake_string("Hello, world!"); asm.store(Opnd::mem(64, SP, 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: adr x16, #8 + 0x4: mov x0, x16 + 0x8: ldnp d8, d25, [x10, #-0x140] + 0xc: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x10: .byte 0x6f, 0x72, 0x6c, 0x64 + 0x14: .byte 0x21, 0x00, 0x00, 0x00 + 0x18: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"50000010e00310aa48656c6c6f2c20776f726c6421000000a00200f8"); } #[test] @@ -1442,7 +2199,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that two instructions were written: LDUR and STUR. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur x0, [x21] + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a00240f8a00200f8"); } #[test] @@ -1454,7 +2215,12 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that three instructions were written: ADD, LDUR, and STUR. - assert_eq!(12, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x21, #0x400 + 0x4: ldur x0, [x0] + 0x8: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a0021091000040f8a00200f8"); } #[test] @@ -1466,7 +2232,13 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that three instructions were written: MOVZ, ADD, LDUR, and STUR. - assert_eq!(16, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #0x1001 + 0x4: add x0, x21, x0, uxtx + 0x8: ldur x0, [x0] + 0xc: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"200082d2a062208b000040f8a00200f8"); } #[test] @@ -1479,7 +2251,11 @@ mod tests { // Assert that only two instructions were written since the value is an // immediate. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #4 + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"800080d2a00200f8"); } #[test] @@ -1492,25 +2268,41 @@ mod tests { // Assert that five instructions were written since the value is not an // immediate and needs to be loaded into a register. - assert_eq!(20, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldr x0, #8 + 0x4: b #0x10 + 0x8: eon x0, x0, x30, ror #0 + 0xc: eon x30, x23, x30, ror #50 + 0x10: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"40000058030000140000fecafecafecaa00200f8"); } #[test] fn test_emit_test_32b_reg_not_bitmask_imm() { let (mut asm, mut cb) = setup_asm(); - let w0 = Opnd::Reg(X0_REG).with_num_bits(32).unwrap(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); asm.test(w0, Opnd::UImm(u32::MAX.into())); // All ones is not encodable with a bitmask immediate, // so this needs one register asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: orr x0, xzr, #0xffffffff + 0x4: tst w0, w0 + "); + assert_snapshot!(cb.hexdump(), @"e07f40b21f00006a"); } #[test] fn test_emit_test_32b_reg_bitmask_imm() { let (mut asm, mut cb) = setup_asm(); - let w0 = Opnd::Reg(X0_REG).with_num_bits(32).unwrap(); + let w0 = Opnd::Reg(X0_REG).with_num_bits(32); asm.test(w0, Opnd::UImm(0x80000001)); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"1f040172"); } #[test] @@ -1520,6 +2312,12 @@ mod tests { let opnd = asm.or(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: orr x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001aa400000f8"); } #[test] @@ -1529,6 +2327,12 @@ mod tests { let opnd = asm.lshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: lsl x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00e87bd3400000f8"); } #[test] @@ -1538,6 +2342,12 @@ mod tests { let opnd = asm.rshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: asr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc4593400000f8"); } #[test] @@ -1547,6 +2357,12 @@ mod tests { let opnd = asm.urshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: lsr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc45d3400000f8"); } #[test] @@ -1557,7 +2373,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 0); // Assert that only one instruction was written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); } #[test] @@ -1568,7 +2385,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 0); // Assert that only one instruction was written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); } #[test] @@ -1579,7 +2397,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that a load and a test instruction were written. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); } #[test] @@ -1590,7 +2412,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 0); // Assert that only one instruction was written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); } #[test] @@ -1601,7 +2424,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that a load and a test instruction were written. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); } #[test] @@ -1612,7 +2439,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that a test instruction is written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #-7"); + assert_snapshot!(cb.hexdump(), @"1ff47df2"); } #[test] @@ -1622,6 +2450,13 @@ mod tests { let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); asm.cmp(shape_opnd, Opnd::UImm(4097)); asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldur w0, [x0, #6] + 0x4: mov x1, #0x1001 + 0x8: cmp w0, w1 + "); + assert_snapshot!(cb.hexdump(), @"006040b8210082d21f00016b"); } #[test] @@ -1631,6 +2466,12 @@ mod tests { let shape_opnd = Opnd::mem(16, Opnd::Reg(X0_REG), 0); asm.store(shape_opnd, Opnd::UImm(4097)); asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #0x1001 + 0x4: sturh w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"300082d210000078"); } #[test] @@ -1640,47 +2481,12 @@ mod tests { let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); asm.store(shape_opnd, Opnd::UImm(4097)); asm.compile_with_num_regs(&mut cb, 2); - } - #[test] - fn test_bcond_straddling_code_pages() { - const LANDING_PAGE: usize = 65; - let mut asm = Assembler::new(0); - let mut cb = CodeBlock::new_dummy_with_freed_pages(vec![0, LANDING_PAGE]); - - // Skip to near the end of the page. Room for two instructions. - cb.set_pos(cb.page_start_pos() + cb.page_end() - 8); - - let end = asm.new_label("end"); - // Start with a conditional jump... - asm.jz(end); - - // A few instructions, enough to cause a page switch. - let sum = asm.add(399.into(), 111.into()); - let xorred = asm.xor(sum, 859.into()); - asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), xorred); - asm.store(Opnd::mem(64, Opnd::Reg(X0_REG), 0), xorred); - - // The branch target. It should be in the landing page. - asm.write_label(end); - asm.cret(xorred); - - // [Bug #19385] - // This used to panic with "The offset must be 19 bits or less." - // due to attempting to lower the `asm.jz` above to a `b.e` with an offset that's > 1 MiB. - let starting_pos = cb.get_write_pos(); - asm.compile_with_num_regs(&mut cb, 2); - let gap = cb.get_write_pos() - starting_pos; - assert!(gap > 0b1111111111111111111); - - let instruction_at_starting_pos: [u8; 4] = unsafe { - std::slice::from_raw_parts(cb.get_ptr(starting_pos).raw_ptr(&cb), 4) - }.try_into().unwrap(); - assert_eq!( - 0b000101 << 26_u32, - u32::from_le_bytes(instruction_at_starting_pos) & (0b111111 << 26_u32), - "starting instruction should be an unconditional branch to the new page (B)" - ); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #0x1001 + 0x4: stur w16, [x0, #6] + "); + assert_snapshot!(cb.hexdump(), @"300082d2106000b8"); } #[test] @@ -1692,10 +2498,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "0b0001ca4b0000f8", " - 0x0: eor x11, x0, x1 - 0x4: stur x11, [x2] + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: eor x0, x0, x1 + 0x4: stur x0, [x2] "); + assert_snapshot!(cb.hexdump(), @"000001ca400000f8"); } #[test] @@ -1729,9 +2536,8 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::mem(64, CFP, 8)); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "618240f8", {" - 0x0: ldur x1, [x19, #8] - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x1, [x19, #8]"); + assert_snapshot!(cb.hexdump(), @"618240f8"); } #[test] @@ -1742,10 +2548,11 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::UImm(0x10000)); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "e1ff9fd2e10370b2", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x1, #0xffff 0x4: orr x1, xzr, #0x10000 - "}); + "); + assert_snapshot!(cb.hexdump(), @"e1ff9fd2e10370b2"); } #[test] @@ -1756,11 +2563,40 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), out); asm.compile_with_num_regs(&mut cb, 2); - assert_disasm!(cb, "8b0280d20c0080d261b18c9a", {" - 0x0: mov x11, #0x14 - 0x4: mov x12, #0 - 0x8: csel x1, x11, x12, lt - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0x14 + 0x4: mov x1, #0 + 0x8: csel x1, x0, x1, lt + "); + assert_snapshot!(cb.hexdump(), @"800280d2010080d201b0819a"); + } + + #[test] + fn test_label_branch_generate_bounds() { + // The immediate in a conditional branch is a 19 bit unsigned integer + // which has a max value of 2^18 - 1. + const IMMEDIATE_MAX_VALUE: usize = 2usize.pow(18) - 1; + + // `IMMEDIATE_MAX_VALUE` number of dummy instructions will be generated + // plus a compare, a jump instruction, and a label. + // Adding page_size to avoid OOM on the last page. + let page_size = unsafe { rb_jit_get_page_size() } as usize; + let memory_required = (IMMEDIATE_MAX_VALUE + 8) * 4 + page_size; + + let mut asm = Assembler::new(); + let mut cb = CodeBlock::new_dummy_sized(memory_required); + + let far_label = asm.new_label("far"); + + asm.cmp(Opnd::Reg(X0_REG), Opnd::UImm(1)); + asm.je(far_label.clone()); + + (0..IMMEDIATE_MAX_VALUE).for_each(|_| { + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::Reg(TEMP_REGS[2])); + }); + + asm.write_label(far_label.clone()); + asm.compile_with_num_regs(&mut cb, 1); } #[test] @@ -1772,26 +2608,174 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), out); asm.compile_with_num_regs(&mut cb, 2); - assert_disasm!(cb, "2b0500b16b0500b1e1030baa", {" - 0x0: adds x11, x9, #1 - 0x4: adds x11, x11, #1 - 0x8: mov x1, x11 - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: adds x0, x9, #1 + 0x4: adds x1, x0, #1 + "); + assert_snapshot!(cb.hexdump(), @"200500b1010400b1"); } #[test] - fn test_mul_with_immediate() { + fn test_store_spilled_byte() { let (mut asm, mut cb) = setup_asm(); - let out = asm.mul(Opnd::Reg(TEMP_REGS[1]), 3.into()); - asm.mov(Opnd::Reg(TEMP_REGS[0]), out); - asm.compile_with_num_regs(&mut cb, 2); + asm.store(Opnd::mem(8, C_RET_OPND, 0), Opnd::mem(8, C_RET_OPND, 8)); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: ldurb w16, [x0, #8] + 0x4: sturb w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"1080403810000038"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_no_cycle() { + let (mut asm, mut cb) = setup_asm(); + + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[0], // mov x0, x0 (optimized away) + C_ARG_OPNDS[1], // mov x1, x1 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0 + 0x4: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"100080d200023fd6"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_single_cycle() { + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[2], // mov x2, x2 (optimized away) + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x15 + 0xc: mov x16, #0 + 0x10: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae1030faa100080d200023fd6"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_two_cycles() { + let (mut asm, mut cb) = setup_asm(); + + // x0 and x1 form a cycle, and x2 and rcx form another cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[0], // mov x1, x0 + C_ARG_OPNDS[3], // mov x2, rcx + C_ARG_OPNDS[2], // mov rcx, x2 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x2 + 0x4: mov x2, x3 + 0x8: mov x3, x15 + 0xc: mov x15, x0 + 0x10: mov x0, x1 + 0x14: mov x1, x15 + 0x18: mov x16, #0 + 0x1c: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0302aae20303aae3030faaef0300aae00301aae1030faa100080d200023fd6"); + } + + #[test] + fn test_ccall_resolve_parallel_moves_large_cycle() { + let (mut asm, mut cb) = setup_asm(); + + // x0, x1, and x2 form a cycle + asm.ccall(0 as _, vec![ + C_ARG_OPNDS[1], // mov x0, x1 + C_ARG_OPNDS[2], // mov x1, x2 + C_ARG_OPNDS[0], // mov x2, x0 + ]); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x2 + 0xc: mov x2, x15 + 0x10: mov x16, #0 + 0x14: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae10302aae2030faa100080d200023fd6"); + } - assert_disasm!(cb, "6b0080d22b7d0b9be1030baa", {" - 0x0: mov x11, #3 - 0x4: mul x11, x9, x11 - 0x8: mov x1, x11 - "}); + #[test] + fn test_split_spilled_lshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd_vreg = asm.load(1.into()); + let out_vreg = asm.lshift(opnd_vreg, Opnd::UImm(1)); + asm.mov(C_RET_OPND, out_vreg); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov x16, #1 + 0x4: stur x16, [x29, #-8] + 0x8: ldur x15, [x29, #-8] + 0xc: lsl x15, x15, #1 + 0x10: stur x15, [x29, #-8] + 0x14: ldur x0, [x29, #-8] + "); + assert_snapshot!(cb.hexdump(), @"300080d2b0831ff8af835ff8eff97fd3af831ff8a0835ff8"); + } + + #[test] + fn test_split_load16_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(16, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x0, #0x200 + 0x4: ldurh w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"0000089100004078"); + } + + #[test] + fn test_split_load32_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(32, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x0, #0x200 + 0x4: ldur w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040b8"); + } + + #[test] + fn test_split_load64_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(64, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: add x0, x0, #0x200 + 0x4: ldur x0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040f8"); } } -*/ diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 5bca786d13..d8d82a09ca 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -1,40 +1,78 @@ +use std::collections::HashMap; use std::fmt; use std::mem::take; -use crate::{cruby::VALUE, hir::FrameState}; -use crate::backend::current::*; +use std::panic; +use std::rc::Rc; +use std::sync::{Arc, Mutex}; +use crate::codegen::local_size_and_idx_to_ep_offset; +use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; +use crate::hir::{Invariant, SideExitReason}; +use crate::options::{TraceExits, debug, get_option}; +use crate::cruby::VALUE; +use crate::payload::IseqVersionRef; +use crate::stats::{exit_counter_ptr, exit_counter_ptr_for_opcode, side_exit_counter, CompileError}; use crate::virtualmem::CodePtr; use crate::asm::{CodeBlock, Label}; -#[cfg(feature = "disasm")] -use crate::options::*; +use crate::state::rb_zjit_record_exit_stack; -pub const EC: Opnd = _EC; -pub const CFP: Opnd = _CFP; -pub const SP: Opnd = _SP; +pub use crate::backend::current::{ + mem_base_reg, + Reg, + EC, CFP, SP, + NATIVE_STACK_PTR, NATIVE_BASE_PTR, + C_ARG_OPNDS, C_RET_REG, C_RET_OPND, +}; -pub const C_ARG_OPNDS: [Opnd; 6] = _C_ARG_OPNDS; -pub const C_RET_OPND: Opnd = _C_RET_OPND; -pub use crate::backend::current::{Reg, C_RET_REG}; +pub static JIT_PRESERVED_REGS: &[Opnd] = &[CFP, SP, EC]; // Memory operand base -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum MemBase { + /// Register: Every Opnd::Mem should have MemBase::Reg as of emit. Reg(u8), + /// Virtual register: Lowered to MemBase::Reg or MemBase::Stack in alloc_regs. VReg(usize), + /// Stack slot: Lowered to MemBase::Reg in scratch_split. + Stack { stack_idx: usize, num_bits: u8 }, } // Memory location -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Hash)] pub struct Mem { // Base register number or instruction index - pub(super) base: MemBase, + pub base: MemBase, // Offset relative to the base pointer - pub(super) disp: i32, + pub disp: i32, // Size in bits - pub(super) num_bits: u8, + pub num_bits: u8, +} + +impl fmt::Display for Mem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.num_bits != 64 { + write!(f, "Mem{}", self.num_bits)?; + } + write!(f, "[")?; + match self.base { + MemBase::Reg(reg_no) => write!(f, "{}", mem_base_reg(reg_no))?, + MemBase::VReg(idx) => write!(f, "v{idx}")?, + MemBase::Stack { stack_idx, num_bits } if num_bits == 64 => write!(f, "Stack[{stack_idx}]")?, + MemBase::Stack { stack_idx, num_bits } => write!(f, "Stack{num_bits}[{stack_idx}]")?, + } + if self.disp != 0 { + let sign = if self.disp > 0 { '+' } else { '-' }; + write!(f, " {sign} ")?; + if self.disp.abs() >= 10 { + write!(f, "0x")?; + } + write!(f, "{:x}", self.disp.abs())?; + } + write!(f, "]") + } } impl fmt::Debug for Mem { @@ -42,7 +80,7 @@ impl fmt::Debug for Mem { write!(fmt, "Mem{}[{:?}", self.num_bits, self.base)?; if self.disp != 0 { let sign = if self.disp > 0 { '+' } else { '-' }; - write!(fmt, " {sign} {}", self.disp)?; + write!(fmt, " {sign} {}", self.disp.abs())?; } write!(fmt, "]") @@ -50,7 +88,7 @@ impl fmt::Debug for Mem { } /// Operand to an IR instruction -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] pub enum Opnd { None, // For insns with no output @@ -68,13 +106,33 @@ pub enum Opnd Reg(Reg), // Machine register } +impl fmt::Display for Opnd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Opnd::*; + match self { + None => write!(f, "None"), + Value(VALUE(value)) if *value < 10 => write!(f, "Value({value:x})"), + Value(VALUE(value)) => write!(f, "Value(0x{value:x})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(f, "v{idx}"), + VReg { idx, num_bits } => write!(f, "VReg{num_bits}(v{idx})"), + Imm(value) if value.abs() < 10 => write!(f, "Imm({value:x})"), + Imm(value) => write!(f, "Imm(0x{value:x})"), + UImm(value) if *value < 10 => write!(f, "{value:x}"), + UImm(value) => write!(f, "0x{value:x}"), + Mem(mem) => write!(f, "{mem}"), + Reg(reg) => write!(f, "{reg}"), + } + } +} + impl fmt::Debug for Opnd { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { use Opnd::*; match self { Self::None => write!(fmt, "None"), Value(val) => write!(fmt, "Value({val:?})"), - VReg { idx, num_bits } => write!(fmt, "Out{num_bits}({idx})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(fmt, "VReg({idx})"), + VReg { idx, num_bits } => write!(fmt, "VReg{num_bits}({idx})"), Imm(signed) => write!(fmt, "{signed:x}_i64"), UImm(unsigned) => write!(fmt, "{unsigned:x}_u64"), // Say Mem and Reg only once @@ -93,8 +151,8 @@ impl Opnd assert!(base_reg.num_bits == 64); Opnd::Mem(Mem { base: MemBase::Reg(base_reg.reg_no), - disp: disp, - num_bits: num_bits, + disp, + num_bits, }) }, @@ -102,17 +160,17 @@ impl Opnd assert!(num_bits <= out_num_bits); Opnd::Mem(Mem { base: MemBase::VReg(idx), - disp: disp, - num_bits: num_bits, + disp, + num_bits, }) }, - _ => unreachable!("memory operand with non-register base") + _ => unreachable!("memory operand with non-register base: {base:?}") } } /// Constructor for constant pointer operand - pub fn const_ptr(ptr: *const u8) -> Self { + pub fn const_ptr<T>(ptr: *const T) -> Self { Opnd::UImm(ptr as u64) } @@ -142,14 +200,15 @@ impl Opnd } } - pub fn with_num_bits(&self, num_bits: u8) -> Option<Opnd> { + /// Return Opnd with a given num_bits if self has num_bits. Panic otherwise. + #[track_caller] + pub fn with_num_bits(&self, num_bits: u8) -> Opnd { assert!(num_bits == 8 || num_bits == 16 || num_bits == 32 || num_bits == 64); match *self { - Opnd::Reg(reg) => Some(Opnd::Reg(reg.with_num_bits(num_bits))), - Opnd::Mem(Mem { base, disp, .. }) => Some(Opnd::Mem(Mem { base, disp, num_bits })), - Opnd::VReg { idx, .. } => Some(Opnd::VReg { idx, num_bits }), - //Opnd::Stack { idx, stack_size, num_locals, sp_offset, reg_mapping, .. } => Some(Opnd::Stack { idx, num_bits, stack_size, num_locals, sp_offset, reg_mapping }), - _ => None, + Opnd::Reg(reg) => Opnd::Reg(reg.with_num_bits(num_bits)), + Opnd::Mem(Mem { base, disp, .. }) => Opnd::Mem(Mem { base, disp, num_bits }), + Opnd::VReg { idx, .. } => Opnd::VReg { idx, num_bits }, + _ => unreachable!("with_num_bits should not be used for: {self:?}"), } } @@ -202,31 +261,6 @@ impl Opnd pub fn match_num_bits(opnds: &[Opnd]) -> u8 { Self::match_num_bits_iter(opnds.iter()) } - - /* - /// Convert Opnd::Stack into RegMapping - pub fn reg_opnd(&self) -> RegOpnd { - self.get_reg_opnd().unwrap() - } - - /// Convert an operand into RegMapping if it's Opnd::Stack - pub fn get_reg_opnd(&self) -> Option<RegOpnd> { - match *self { - Opnd::Stack { idx, stack_size, num_locals, .. } => Some( - if let Some(num_locals) = num_locals { - let last_idx = stack_size as i32 + VM_ENV_DATA_SIZE as i32 - 1; - assert!(last_idx <= idx, "Local index {} must be >= last local index {}", idx, last_idx); - assert!(idx <= last_idx + num_locals as i32, "Local index {} must be < last local index {} + local size {}", idx, last_idx, num_locals); - RegOpnd::Local((last_idx + num_locals as i32 - idx) as u8) - } else { - assert!(idx < stack_size as i32); - RegOpnd::Stack((stack_size as i32 - idx - 1) as u8) - } - ), - _ => None, - } - } - */ } impl From<usize> for Opnd { @@ -265,6 +299,14 @@ impl From<VALUE> for Opnd { } } +/// Context for a side exit. If `SideExit` matches, it reuses the same code. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SideExit { + pub pc: Opnd, + pub stack: Vec<Opnd>, + pub locals: Vec<Opnd>, +} + /// Branch target (something that we can jump to) /// for branch instructions #[derive(Clone, Debug)] @@ -272,12 +314,15 @@ pub enum Target { /// Pointer to a piece of ZJIT-generated code CodePtr(CodePtr), - // Side exit with a counter - SideExit(FrameState), - /// Pointer to a side exit code - SideExitPtr(CodePtr), /// A label within the generated code Label(Label), + /// Side exit to the interpreter + SideExit { + /// Context used for compiling the side exit + exit: SideExit, + /// We use this to increment exit counters + reason: SideExitReason, + }, } impl Target @@ -292,7 +337,6 @@ impl Target pub fn unwrap_code_ptr(&self) -> CodePtr { match self { Target::CodePtr(ptr) => *ptr, - Target::SideExitPtr(ptr) => *ptr, _ => unreachable!("trying to unwrap {:?} into code ptr", self) } } @@ -304,9 +348,10 @@ impl From<CodePtr> for Target { } } -type PosMarkerFn = Box<dyn Fn(CodePtr, &CodeBlock)>; +type PosMarkerFn = Rc<dyn Fn(CodePtr, &CodeBlock)>; /// ZJIT Low-level IR instruction +#[derive(Clone)] pub enum Insn { /// Add two operands together, and return the result as a new operand. Add { left: Opnd, right: Opnd, out: Opnd }, @@ -347,7 +392,9 @@ pub enum Insn { // C function call with N arguments (variadic) CCall { opnds: Vec<Opnd>, - fptr: *const u8, + /// The function pointer to be called. This should be Opnd::const_ptr + /// (Opnd::UImm) in most cases. gen_entry_trampoline() uses Opnd::Reg. + fptr: Opnd, /// Optional PosMarker to remember the start address of the C call. /// It's embedded here to insert the PosMarker after push instructions /// that are split from this CCall on alloc_regs(). @@ -387,10 +434,10 @@ pub enum Insn { CSelZ { truthy: Opnd, falsy: Opnd, out: Opnd }, /// Set up the frame stack as necessary per the architecture. - FrameSetup, + FrameSetup { preserved: &'static [Opnd], slot_count: usize }, /// Tear down the frame stack as necessary per the architecture. - FrameTeardown, + FrameTeardown { preserved: &'static [Opnd], }, // Atomically increment a counter // Input: memory operand, increment value @@ -467,9 +514,9 @@ pub enum Insn { /// Shift a value left by a certain amount. LShift { opnd: Opnd, shift: Opnd, out: Opnd }, - /// A set of parallel moves into registers. + /// A set of parallel moves into registers or memory. /// The backend breaks cycles if there are any cycles between moves. - ParallelMov { moves: Vec<(Reg, Opnd)> }, + ParallelMov { moves: Vec<(Opnd, Opnd)> }, // A low-level mov instruction. It accepts two operands. Mov { dest: Opnd, src: Opnd }, @@ -483,9 +530,13 @@ pub enum Insn { // binary OR operation. Or { left: Opnd, right: Opnd, out: Opnd }, - /// Pad nop instructions to accommodate Op::Jmp in case the block or the insn - /// is invalidated. - PadInvalPatch, + /// Patch point that will be rewritten to a jump to a side exit on invalidation. + PatchPoint { target: Target, invariant: Invariant, version: IseqVersionRef }, + + /// Make sure the last PatchPoint has enough space to insert a jump. + /// We insert this instruction at the end of each block so that the jump + /// will not overwrite the next block or a side exit. + PadPatchPoint, // Mark a position in the generated code PosMarker(PosMarkerFn), @@ -516,13 +567,13 @@ pub enum Insn { impl Insn { /// Create an iterator that will yield a non-mutable reference to each /// operand in turn for this instruction. - pub(super) fn opnd_iter(&self) -> InsnOpndIterator { + pub(super) fn opnd_iter(&self) -> InsnOpndIterator<'_> { InsnOpndIterator::new(self) } /// Create an iterator that will yield a mutable reference to each operand /// in turn for this instruction. - pub(super) fn opnd_iter_mut(&mut self) -> InsnOpndMutIterator { + pub(super) fn opnd_iter_mut(&mut self) -> InsnOpndMutIterator<'_> { InsnOpndMutIterator::new(self) } @@ -539,12 +590,13 @@ impl Insn { Insn::Jne(target) | Insn::Jnz(target) | Insn::Jo(target) | - Insn::Jz(target) | - Insn::Label(target) | Insn::JoMul(target) | + Insn::Jz(target) | Insn::Joz(_, target) | Insn::Jonz(_, target) | - Insn::LeaJumpTarget { target, .. } => { + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { Some(target) } _ => None, @@ -576,8 +628,8 @@ impl Insn { Insn::CSelNE { .. } => "CSelNE", Insn::CSelNZ { .. } => "CSelNZ", Insn::CSelZ { .. } => "CSelZ", - Insn::FrameSetup => "FrameSetup", - Insn::FrameTeardown => "FrameTeardown", + Insn::FrameSetup { .. } => "FrameSetup", + Insn::FrameTeardown { .. } => "FrameTeardown", Insn::IncrCounter { .. } => "IncrCounter", Insn::Jbe(_) => "Jbe", Insn::Jb(_) => "Jb", @@ -606,7 +658,8 @@ impl Insn { Insn::Mov { .. } => "Mov", Insn::Not { .. } => "Not", Insn::Or { .. } => "Or", - Insn::PadInvalPatch => "PadEntryExit", + Insn::PatchPoint { .. } => "PatchPoint", + Insn::PadPatchPoint => "PadPatchPoint", Insn::PosMarker(_) => "PosMarker", Insn::RShift { .. } => "RShift", Insn::Store { .. } => "Store", @@ -697,8 +750,13 @@ impl Insn { Insn::Jne(target) | Insn::Jnz(target) | Insn::Jo(target) | + Insn::JoMul(target) | Insn::Jz(target) | - Insn::LeaJumpTarget { target, .. } => Some(target), + Insn::Joz(_, target) | + Insn::Jonz(_, target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => Some(target), _ => None } } @@ -731,29 +789,71 @@ impl<'a> Iterator for InsnOpndIterator<'a> { fn next(&mut self) -> Option<Self::Item> { match self.insn { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { + let stack_idx = self.idx; + if stack_idx < stack.len() { + let opnd = &stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = self.idx - stack.len(); + if local_idx < locals.len() { + let opnd = &locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + } + None + } + + Insn::Joz(opnd, target) | + Insn::Jonz(opnd, target) => { + if self.idx == 0 { + self.idx += 1; + return Some(opnd); + } + + if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { + let stack_idx = self.idx - 1; + if stack_idx < stack.len() { + let opnd = &stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = stack_idx - stack.len(); + if local_idx < locals.len() { + let opnd = &locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + } + None + } + Insn::BakeString(_) | Insn::Breakpoint | Insn::Comment(_) | Insn::CPop { .. } | Insn::CPopAll | Insn::CPushAll | - Insn::FrameSetup | - Insn::FrameTeardown | - Insn::Jbe(_) | - Insn::Jb(_) | - Insn::Je(_) | - Insn::Jl(_) | - Insn::Jg(_) | - Insn::Jge(_) | - Insn::Jmp(_) | - Insn::Jne(_) | - Insn::Jnz(_) | - Insn::Jo(_) | - Insn::JoMul(_) | - Insn::Jz(_) | - Insn::Label(_) | - Insn::LeaJumpTarget { .. } | - Insn::PadInvalPatch | + Insn::PadPatchPoint | Insn::PosMarker(_) => None, Insn::CPopInto(opnd) | @@ -764,8 +864,6 @@ impl<'a> Iterator for InsnOpndIterator<'a> { Insn::LiveReg { opnd, .. } | Insn::Load { opnd, .. } | Insn::LoadSExt { opnd, .. } | - Insn::Joz(opnd, _) | - Insn::Jonz(opnd, _) | Insn::Not { opnd, .. } => { match self.idx { 0 => { @@ -820,14 +918,29 @@ impl<'a> Iterator for InsnOpndIterator<'a> { } }, Insn::ParallelMov { moves } => { - if self.idx < moves.len() { - let opnd = &moves[self.idx].1; + if self.idx < moves.len() * 2 { + let move_idx = self.idx / 2; + let opnd = if self.idx % 2 == 0 { + &moves[move_idx].0 + } else { + &moves[move_idx].1 + }; self.idx += 1; Some(opnd) } else { None } }, + Insn::FrameSetup { preserved, .. } | + Insn::FrameTeardown { preserved } => { + if self.idx < preserved.len() { + let opnd = &preserved[self.idx]; + self.idx += 1; + Some(opnd) + } else { + None + } + } } } } @@ -845,29 +958,73 @@ impl<'a> InsnOpndMutIterator<'a> { pub(super) fn next(&mut self) -> Option<&mut Opnd> { match self.insn { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { + let stack_idx = self.idx; + if stack_idx < stack.len() { + let opnd = &mut stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = self.idx - stack.len(); + if local_idx < locals.len() { + let opnd = &mut locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + } + None + } + + Insn::Joz(opnd, target) | + Insn::Jonz(opnd, target) => { + if self.idx == 0 { + self.idx += 1; + return Some(opnd); + } + + if let Target::SideExit { exit: SideExit { stack, locals, .. }, .. } = target { + let stack_idx = self.idx - 1; + if stack_idx < stack.len() { + let opnd = &mut stack[stack_idx]; + self.idx += 1; + return Some(opnd); + } + + let local_idx = stack_idx - stack.len(); + if local_idx < locals.len() { + let opnd = &mut locals[local_idx]; + self.idx += 1; + return Some(opnd); + } + } + None + } + Insn::BakeString(_) | Insn::Breakpoint | Insn::Comment(_) | Insn::CPop { .. } | Insn::CPopAll | Insn::CPushAll | - Insn::FrameSetup | - Insn::FrameTeardown | - Insn::Jbe(_) | - Insn::Jb(_) | - Insn::Je(_) | - Insn::Jl(_) | - Insn::Jg(_) | - Insn::Jge(_) | - Insn::Jmp(_) | - Insn::Jne(_) | - Insn::Jnz(_) | - Insn::Jo(_) | - Insn::JoMul(_) | - Insn::Jz(_) | - Insn::Label(_) | - Insn::LeaJumpTarget { .. } | - Insn::PadInvalPatch | + Insn::FrameSetup { .. } | + Insn::FrameTeardown { .. } | + Insn::PadPatchPoint | Insn::PosMarker(_) => None, Insn::CPopInto(opnd) | @@ -878,8 +1035,6 @@ impl<'a> InsnOpndMutIterator<'a> { Insn::LiveReg { opnd, .. } | Insn::Load { opnd, .. } | Insn::LoadSExt { opnd, .. } | - Insn::Joz(opnd, _) | - Insn::Jonz(opnd, _) | Insn::Not { opnd, .. } => { match self.idx { 0 => { @@ -934,8 +1089,13 @@ impl<'a> InsnOpndMutIterator<'a> { } }, Insn::ParallelMov { moves } => { - if self.idx < moves.len() { - let opnd = &mut moves[self.idx].1; + if self.idx < moves.len() * 2 { + let move_idx = self.idx / 2; + let opnd = if self.idx % 2 == 0 { + &mut moves[move_idx].0 + } else { + &mut moves[move_idx].1 + }; self.idx += 1; Some(opnd) } else { @@ -952,6 +1112,9 @@ impl fmt::Debug for Insn { // Print list of operands let mut opnd_iter = self.opnd_iter(); + if let Insn::FrameSetup { slot_count, .. } = self { + write!(fmt, "{slot_count}")?; + } if let Some(first_opnd) = opnd_iter.next() { write!(fmt, "{first_opnd:?}")?; } @@ -994,6 +1157,81 @@ impl LiveRange { } } +/// StackState manages which stack slots are used by which VReg +pub struct StackState { + /// The maximum number of spilled VRegs at a time + stack_size: usize, + /// Map from index at the C stack for spilled VRegs to Some(vreg_idx) if allocated + stack_slots: Vec<Option<usize>>, + /// Copy of Assembler::stack_base_idx. Used for calculating stack slot offsets. + stack_base_idx: usize, +} + +impl StackState { + /// Initialize a stack allocator + pub(super) fn new(stack_base_idx: usize) -> Self { + StackState { + stack_size: 0, + stack_slots: vec![], + stack_base_idx, + } + } + + /// Allocate a stack slot for a given vreg_idx + fn alloc_stack(&mut self, vreg_idx: usize) -> Opnd { + for stack_idx in 0..self.stack_size { + if self.stack_slots[stack_idx].is_none() { + self.stack_slots[stack_idx] = Some(vreg_idx); + return Opnd::mem(64, NATIVE_BASE_PTR, self.stack_idx_to_disp(stack_idx)); + } + } + // Every stack slot is in use. Allocate a new stack slot. + self.stack_size += 1; + self.stack_slots.push(Some(vreg_idx)); + Opnd::mem(64, NATIVE_BASE_PTR, self.stack_idx_to_disp(self.stack_slots.len() - 1)) + } + + /// Deallocate a stack slot for a given disp + fn dealloc_stack(&mut self, disp: i32) { + let stack_idx = self.disp_to_stack_idx(disp); + if self.stack_slots[stack_idx].is_some() { + self.stack_slots[stack_idx] = None; + } + } + + /// Convert the `disp` of a stack slot operand to the stack index + fn disp_to_stack_idx(&self, disp: i32) -> usize { + (-disp / SIZEOF_VALUE_I32) as usize - self.stack_base_idx - 1 + } + + /// Convert a stack index to the `disp` of the stack slot + fn stack_idx_to_disp(&self, stack_idx: usize) -> i32 { + (self.stack_base_idx + stack_idx + 1) as i32 * -SIZEOF_VALUE_I32 + } + + /// Convert Mem to MemBase::Stack + fn mem_to_stack_membase(&self, mem: Mem) -> MemBase { + match mem { + Mem { base: MemBase::Reg(reg_no), disp, num_bits } if NATIVE_BASE_PTR.unwrap_reg().reg_no == reg_no => { + let stack_idx = self.disp_to_stack_idx(disp); + MemBase::Stack { stack_idx, num_bits } + } + _ => unreachable!(), + } + } + + /// Convert MemBase::Stack to Mem + pub(super) fn stack_membase_to_mem(&self, membase: MemBase) -> Mem { + match membase { + MemBase::Stack { stack_idx, num_bits } => { + let disp = self.stack_idx_to_disp(stack_idx); + Mem { base: MemBase::Reg(NATIVE_BASE_PTR.unwrap_reg().reg_no), disp, num_bits } + } + _ => unreachable!(), + } + } +} + /// RegisterPool manages which registers are used by which VReg struct RegisterPool { /// List of registers that can be allocated @@ -1006,45 +1244,54 @@ struct RegisterPool { /// The number of live registers. /// Provides a quick way to query `pool.filter(|r| r.is_some()).count()` live_regs: usize, + + /// Fallback to let StackState allocate stack slots when RegisterPool runs out of registers. + stack_state: StackState, } impl RegisterPool { /// Initialize a register pool - fn new(regs: Vec<Reg>) -> Self { + fn new(regs: Vec<Reg>, stack_base_idx: usize) -> Self { let pool = vec![None; regs.len()]; RegisterPool { regs, pool, live_regs: 0, + stack_state: StackState::new(stack_base_idx), } } /// Mutate the pool to indicate that the register at the index /// has been allocated and is live. - fn alloc_reg(&mut self, vreg_idx: usize) -> Option<Reg> { + fn alloc_opnd(&mut self, vreg_idx: usize) -> Opnd { for (reg_idx, reg) in self.regs.iter().enumerate() { if self.pool[reg_idx].is_none() { self.pool[reg_idx] = Some(vreg_idx); self.live_regs += 1; - return Some(*reg); + return Opnd::Reg(*reg); } } - None + self.stack_state.alloc_stack(vreg_idx) } /// Allocate a specific register - fn take_reg(&mut self, reg: &Reg, vreg_idx: usize) -> Reg { + fn take_reg(&mut self, reg: &Reg, vreg_idx: usize) -> Opnd { let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no) .unwrap_or_else(|| panic!("Unable to find register: {}", reg.reg_no)); - assert_eq!(self.pool[reg_idx], None, "register already allocated"); + assert_eq!(self.pool[reg_idx], None, "register already allocated for VReg({:?})", self.pool[reg_idx]); self.pool[reg_idx] = Some(vreg_idx); self.live_regs += 1; - *reg + Opnd::Reg(*reg) } // Mutate the pool to indicate that the given register is being returned // as it is no longer used by the instruction that previously held it. - fn dealloc_reg(&mut self, reg: &Reg) { + fn dealloc_opnd(&mut self, opnd: &Opnd) { + if let Opnd::Mem(Mem { disp, .. }) = *opnd { + return self.stack_state.dealloc_stack(disp); + } + + let reg = opnd.unwrap_reg(); let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no) .unwrap_or_else(|| panic!("Unable to find register: {}", reg.reg_no)); if self.pool[reg_idx].is_some() { @@ -1081,6 +1328,7 @@ const ASSEMBLER_INSNS_CAPACITY: usize = 256; /// Object into which we assemble instructions to be /// optimized and lowered +#[derive(Clone)] pub struct Assembler { pub(super) insns: Vec<Insn>, @@ -1090,83 +1338,97 @@ pub struct Assembler { /// Names of labels pub(super) label_names: Vec<String>, - /* - /// Context for generating the current insn - pub ctx: Context, + /// If true, `push_insn` is allowed to use scratch registers. + /// On `compile`, it also disables the backend's use of them. + pub(super) accept_scratch_reg: bool, - /// The current ISEQ's local table size. asm.local_opnd() uses this, and it's - /// sometimes hard to pass this value, e.g. asm.spill_regs() in asm.ccall(). - /// - /// `None` means we're not assembling for an ISEQ, or that the local size is - /// not relevant. - pub(super) num_locals: Option<u32>, + /// The Assembler can use NATIVE_BASE_PTR + stack_base_idx as the + /// first stack slot in case it needs to allocate memory. This is + /// equal to the number of spilled basic block arguments. + pub(super) stack_base_idx: usize, - /// Side exit caches for each SideExitContext - pub(super) side_exits: HashMap<SideExitContext, CodePtr>, - - /// PC for Target::SideExit - side_exit_pc: Option<*mut VALUE>, - - /// Stack size for Target::SideExit - side_exit_stack_size: Option<u8>, - - /// If true, the next ccall() should verify its leafness - leaf_ccall: bool, - */ + /// If Some, the next ccall should verify its leafness + leaf_ccall_stack_size: Option<usize> } impl Assembler { - /// Create an Assembler + /// Create an Assembler with defaults pub fn new() -> Self { - Self::new_with_label_names(Vec::default(), 0) + Self { + insns: Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY), + live_ranges: Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY), + label_names: Vec::default(), + accept_scratch_reg: false, + stack_base_idx: 0, + leaf_ccall_stack_size: None, + } } - /* - /// Create an Assembler for ISEQ-specific code. - /// It includes all inline code and some outlined code like side exits and stubs. - pub fn new(num_locals: u32) -> Self { - Self::new_with_label_names(Vec::default(), HashMap::default(), Some(num_locals)) + /// Create an Assembler, reserving a specified number of stack slots + pub fn new_with_stack_slots(stack_base_idx: usize) -> Self { + Self { stack_base_idx, ..Self::new() } } - /// Create an Assembler for outlined code that are not specific to any ISEQ, - /// e.g. trampolines that are shared globally. - pub fn new_without_iseq() -> Self { - Self::new_with_label_names(Vec::default(), HashMap::default(), None) + /// Create an Assembler that allows the use of scratch registers. + /// This should be called only through [`Self::new_with_scratch_reg`]. + pub(super) fn new_with_accept_scratch_reg(accept_scratch_reg: bool) -> Self { + Self { accept_scratch_reg, ..Self::new() } } - */ - /// Create an Assembler with parameters that are populated by another Assembler instance. - /// This API is used for copying an Assembler for the next compiler pass. - pub fn new_with_label_names(label_names: Vec<String>, num_vregs: usize) -> Self { - let mut live_ranges = Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY); - live_ranges.resize(num_vregs, LiveRange { start: None, end: None }); + /// Create an Assembler with parameters of another Assembler and empty instructions. + /// Compiler passes build a next Assembler with this API and insert new instructions to it. + pub(super) fn new_with_asm(old_asm: &Assembler) -> Self { + let mut asm = Self { + label_names: old_asm.label_names.clone(), + accept_scratch_reg: old_asm.accept_scratch_reg, + stack_base_idx: old_asm.stack_base_idx, + ..Self::new() + }; + // Bump the initial VReg index to allow the use of the VRegs for the old Assembler + asm.live_ranges.resize(old_asm.live_ranges.len(), LiveRange { start: None, end: None }); + asm + } - Self { - insns: Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY), - live_ranges, - label_names, + /// Return true if `opnd` is or depends on `reg` + pub fn has_reg(opnd: Opnd, reg: Reg) -> bool { + match opnd { + Opnd::Reg(opnd_reg) => opnd_reg == reg, + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) => reg_no == reg.reg_no, + _ => false, } } - /* - /// Get the list of registers that can be used for stack temps. - pub fn get_temp_regs2() -> &'static [Reg] { - let num_regs = get_option!(num_temp_regs); - &TEMP_REGS[0..num_regs] + pub fn instruction_iterator(&mut self) -> InsnIter { + let insns = take(&mut self.insns); + InsnIter { + old_insns_iter: insns.into_iter(), + peeked: None, + index: 0, + } } - /// Get the number of locals for the ISEQ being compiled - pub fn get_num_locals(&self) -> Option<u32> { - self.num_locals + pub fn expect_leaf_ccall(&mut self, stack_size: usize) { + self.leaf_ccall_stack_size = Some(stack_size); } - /// Set a context for generating side exits - pub fn set_side_exit_context(&mut self, pc: *mut VALUE, stack_size: u8) { - self.side_exit_pc = Some(pc); - self.side_exit_stack_size = Some(stack_size); + fn set_stack_canary(&mut self) -> Option<Opnd> { + if cfg!(feature = "runtime_checks") { + if let Some(stack_size) = self.leaf_ccall_stack_size.take() { + let canary_addr = self.lea(Opnd::mem(64, SP, (stack_size as i32) * SIZEOF_VALUE_I32)); + let canary_opnd = Opnd::mem(64, canary_addr, 0); + self.mov(canary_opnd, vm_stack_canary().into()); + return Some(canary_opnd) + } + } + None + } + + fn clear_stack_canary(&mut self, canary_opnd: Option<Opnd>){ + if let Some(canary_opnd) = canary_opnd { + self.store(canary_opnd, 0.into()); + }; } - */ /// Build an Opnd::VReg and initialize its LiveRange pub(super) fn new_vreg(&mut self, num_bits: u8) -> Opnd { @@ -1178,7 +1440,7 @@ impl Assembler /// Append an instruction onto the current list of instructions and update /// the live ranges of any instructions whose outputs are being used as /// operands to this instruction. - pub fn push_insn(&mut self, mut insn: Insn) { + pub fn push_insn(&mut self, insn: Insn) { // Index of this instruction let insn_idx = self.insns.len(); @@ -1190,8 +1452,8 @@ impl Assembler } // If we find any VReg from previous instructions, extend the live range to insn_idx - let mut opnd_iter = insn.opnd_iter_mut(); - while let Some(opnd) = opnd_iter.next() { + let opnd_iter = insn.opnd_iter(); + for opnd in opnd_iter { match *opnd { Opnd::VReg { idx, .. } | Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { @@ -1203,26 +1465,16 @@ impl Assembler } } - self.insns.push(insn); - } - - /* - /// Get a cached side exit, wrapping a counter if specified - pub fn get_side_exit(&mut self, side_exit_context: &SideExitContext, counter: Option<Counter>, ocb: &mut OutlinedCb) -> Option<CodePtr> { - // Get a cached side exit - let side_exit = match self.side_exits.get(&side_exit_context) { - None => { - let exit_code = gen_outlined_exit(side_exit_context.pc, self.num_locals.unwrap(), &side_exit_context.get_ctx(), ocb)?; - self.side_exits.insert(*side_exit_context, exit_code); - exit_code + // If this Assembler should not accept scratch registers, assert no use of them. + if !self.accept_scratch_reg { + let opnd_iter = insn.opnd_iter(); + for opnd in opnd_iter { + assert!(!Self::has_scratch_reg(*opnd), "should not use scratch register: {opnd:?}"); } - Some(code_ptr) => *code_ptr, - }; + } - // Wrap a counter if needed - gen_counted_exit(side_exit_context.pc, side_exit, ocb, counter) + self.insns.push(insn); } - */ /// Create a new label instance that we can jump to pub fn new_label(&mut self, name: &str) -> Target @@ -1234,177 +1486,25 @@ impl Assembler Target::Label(label) } - /* - /// Convert Opnd::Stack to Opnd::Mem or Opnd::Reg - pub fn lower_stack_opnd(&self, opnd: &Opnd) -> Opnd { - // Convert Opnd::Stack to Opnd::Mem - fn mem_opnd(opnd: &Opnd) -> Opnd { - if let Opnd::Stack { idx, sp_offset, num_bits, .. } = *opnd { - incr_counter!(temp_mem_opnd); - Opnd::mem(num_bits, SP, (sp_offset as i32 - idx - 1) * SIZEOF_VALUE_I32) - } else { - unreachable!() - } - } - - // Convert Opnd::Stack to Opnd::Reg - fn reg_opnd(opnd: &Opnd, reg_idx: usize) -> Opnd { - let regs = Assembler::get_temp_regs2(); - if let Opnd::Stack { num_bits, .. } = *opnd { - incr_counter!(temp_reg_opnd); - Opnd::Reg(regs[reg_idx]).with_num_bits(num_bits).unwrap() - } else { - unreachable!() - } - } - - match opnd { - Opnd::Stack { reg_mapping, .. } => { - if let Some(reg_idx) = reg_mapping.unwrap().get_reg(opnd.reg_opnd()) { - reg_opnd(opnd, reg_idx) - } else { - mem_opnd(opnd) - } - } - _ => unreachable!(), - } - } - - /// Allocate a register to a stack temp if available. - pub fn alloc_reg(&mut self, mapping: RegOpnd) { - // Allocate a register if there's no conflict. - let mut reg_mapping = self.ctx.get_reg_mapping(); - if reg_mapping.alloc_reg(mapping) { - self.set_reg_mapping(reg_mapping); - } - } - - /// Erase local variable type information - /// eg: because of a call we can't track - pub fn clear_local_types(&mut self) { - asm_comment!(self, "clear local variable types"); - self.ctx.clear_local_types(); - } - - /// Repurpose stack temp registers to the corresponding locals for arguments - pub fn map_temp_regs_to_args(&mut self, callee_ctx: &mut Context, argc: i32) -> Vec<RegOpnd> { - let mut callee_reg_mapping = callee_ctx.get_reg_mapping(); - let mut mapped_temps = vec![]; - - for arg_idx in 0..argc { - let stack_idx: u8 = (self.ctx.get_stack_size() as i32 - argc + arg_idx).try_into().unwrap(); - let temp_opnd = RegOpnd::Stack(stack_idx); - - // For each argument, if the stack temp for it has a register, - // let the callee use the register for the local variable. - if let Some(reg_idx) = self.ctx.get_reg_mapping().get_reg(temp_opnd) { - let local_opnd = RegOpnd::Local(arg_idx.try_into().unwrap()); - callee_reg_mapping.set_reg(local_opnd, reg_idx); - mapped_temps.push(temp_opnd); - } - } - - asm_comment!(self, "local maps: {:?}", callee_reg_mapping); - callee_ctx.set_reg_mapping(callee_reg_mapping); - mapped_temps - } - - /// Spill all live registers to the stack - pub fn spill_regs(&mut self) { - self.spill_regs_except(&vec![]); - } - - /// Spill all live registers except `ignored_temps` to the stack - pub fn spill_regs_except(&mut self, ignored_temps: &Vec<RegOpnd>) { - // Forget registers above the stack top - let mut reg_mapping = self.ctx.get_reg_mapping(); - for stack_idx in self.ctx.get_stack_size()..MAX_CTX_TEMPS as u8 { - reg_mapping.dealloc_reg(RegOpnd::Stack(stack_idx)); - } - self.set_reg_mapping(reg_mapping); - - // If no registers are in use, skip all checks - if self.ctx.get_reg_mapping() == RegMapping::default() { - return; - } - - // Collect stack temps to be spilled - let mut spilled_opnds = vec![]; - for stack_idx in 0..u8::min(MAX_CTX_TEMPS as u8, self.ctx.get_stack_size()) { - let reg_opnd = RegOpnd::Stack(stack_idx); - if !ignored_temps.contains(®_opnd) && reg_mapping.dealloc_reg(reg_opnd) { - let idx = self.ctx.get_stack_size() - 1 - stack_idx; - let spilled_opnd = self.stack_opnd(idx.into()); - spilled_opnds.push(spilled_opnd); - reg_mapping.dealloc_reg(spilled_opnd.reg_opnd()); - } - } - - // Collect locals to be spilled - for local_idx in 0..MAX_CTX_TEMPS as u8 { - if reg_mapping.dealloc_reg(RegOpnd::Local(local_idx)) { - let first_local_ep_offset = self.num_locals.unwrap() + VM_ENV_DATA_SIZE - 1; - let ep_offset = first_local_ep_offset - local_idx as u32; - let spilled_opnd = self.local_opnd(ep_offset); - spilled_opnds.push(spilled_opnd); - reg_mapping.dealloc_reg(spilled_opnd.reg_opnd()); - } - } - - // Spill stack temps and locals - if !spilled_opnds.is_empty() { - asm_comment!(self, "spill_regs: {:?} -> {:?}", self.ctx.get_reg_mapping(), reg_mapping); - for &spilled_opnd in spilled_opnds.iter() { - self.spill_reg(spilled_opnd); - } - self.ctx.set_reg_mapping(reg_mapping); - } - } - - /// Spill a stack temp from a register to the stack - pub fn spill_reg(&mut self, opnd: Opnd) { - assert_ne!(self.ctx.get_reg_mapping().get_reg(opnd.reg_opnd()), None); - - // Use different RegMappings for dest and src operands - let reg_mapping = self.ctx.get_reg_mapping(); - let mut mem_mappings = reg_mapping; - mem_mappings.dealloc_reg(opnd.reg_opnd()); - - // Move the stack operand from a register to memory - match opnd { - Opnd::Stack { idx, num_bits, stack_size, num_locals, sp_offset, .. } => { - self.mov( - Opnd::Stack { idx, num_bits, stack_size, num_locals, sp_offset, reg_mapping: Some(mem_mappings) }, - Opnd::Stack { idx, num_bits, stack_size, num_locals, sp_offset, reg_mapping: Some(reg_mapping) }, - ); - } - _ => unreachable!(), - } - incr_counter!(temp_spill); - } - - /// Update which stack temps are in a register - pub fn set_reg_mapping(&mut self, reg_mapping: RegMapping) { - if self.ctx.get_reg_mapping() != reg_mapping { - asm_comment!(self, "reg_mapping: {:?} -> {:?}", self.ctx.get_reg_mapping(), reg_mapping); - self.ctx.set_reg_mapping(reg_mapping); - } - } - */ - - // Shuffle register moves, sometimes adding extra moves using SCRATCH_REG, + // Shuffle register moves, sometimes adding extra moves using scratch_reg, // so that they will not rewrite each other before they are used. - pub fn resolve_parallel_moves(old_moves: &Vec<(Reg, Opnd)>) -> Vec<(Reg, Opnd)> { + pub fn resolve_parallel_moves(old_moves: &[(Opnd, Opnd)], scratch_opnd: Option<Opnd>) -> Option<Vec<(Opnd, Opnd)>> { // Return the index of a move whose destination is not used as a source if any. - fn find_safe_move(moves: &Vec<(Reg, Opnd)>) -> Option<usize> { - moves.iter().enumerate().find(|&(_, &(dest_reg, _))| { - moves.iter().all(|&(_, src_opnd)| src_opnd != Opnd::Reg(dest_reg)) + fn find_safe_move(moves: &[(Opnd, Opnd)]) -> Option<usize> { + moves.iter().enumerate().find(|&(_, &(dst, src))| { + // Check if `dst` is used in other moves. If `dst` is not used elsewhere, it's safe to write into `dst` now. + moves.iter().filter(|&&other_move| other_move != (dst, src)).all(|&(other_dst, other_src)| + match dst { + Opnd::Reg(reg) => !Assembler::has_reg(other_dst, reg) && !Assembler::has_reg(other_src, reg), + _ => other_dst != dst && other_src != dst, + } + ) }).map(|(index, _)| index) } // Remove moves whose source and destination are the same - let mut old_moves: Vec<(Reg, Opnd)> = old_moves.clone().into_iter() - .filter(|&(reg, opnd)| Opnd::Reg(reg) != opnd).collect(); + let mut old_moves: Vec<(Opnd, Opnd)> = old_moves.iter().copied() + .filter(|&(dst, src)| dst != src).collect(); let mut new_moves = vec![]; while !old_moves.is_empty() { @@ -1413,70 +1513,62 @@ impl Assembler new_moves.push(old_moves.remove(index)); } - // No safe move. Load the source of one move into SCRATCH_REG, and - // then load SCRATCH_REG into the destination when it's safe. + // No safe move. Load the source of one move into scratch_opnd, and + // then load scratch_opnd into the destination when it's safe. if !old_moves.is_empty() { - // Make sure it's safe to use SCRATCH_REG - assert!(old_moves.iter().all(|&(_, opnd)| opnd != Opnd::Reg(Assembler::SCRATCH_REG))); - - // Move SCRATCH <- opnd, and delay reg <- SCRATCH - let (reg, opnd) = old_moves.remove(0); - new_moves.push((Assembler::SCRATCH_REG, opnd)); - old_moves.push((reg, Opnd::Reg(Assembler::SCRATCH_REG))); + // If scratch_opnd is None, return None and leave it to *_split_with_scratch_regs to resolve it. + let scratch_opnd = scratch_opnd?; + let scratch_reg = scratch_opnd.unwrap_reg(); + // Make sure it's safe to use scratch_reg + assert!(old_moves.iter().all(|&(dst, src)| !Self::has_reg(dst, scratch_reg) && !Self::has_reg(src, scratch_reg))); + + // Move scratch_opnd <- src, and delay dst <- scratch_opnd + let (dst, src) = old_moves.remove(0); + new_moves.push((scratch_opnd, src)); + old_moves.push((dst, scratch_opnd)); } } - new_moves + Some(new_moves) } /// Sets the out field on the various instructions that require allocated /// registers because their output is used as the operand on a subsequent /// instruction. This is our implementation of the linear scan algorithm. - pub(super) fn alloc_regs(mut self, regs: Vec<Reg>) -> Assembler { - // Dump live registers for register spill debugging. - fn dump_live_regs(insns: Vec<Insn>, live_ranges: Vec<LiveRange>, num_regs: usize, spill_index: usize) { - // Convert live_ranges to live_regs: the number of live registers at each index - let mut live_regs: Vec<usize> = vec![]; - for insn_idx in 0..insns.len() { - let live_count = live_ranges.iter().filter(|range| range.start() <= insn_idx && insn_idx <= range.end()).count(); - live_regs.push(live_count); - } - - // Dump insns along with live registers - for (insn_idx, insn) in insns.iter().enumerate() { - eprint!("{:3} ", if spill_index == insn_idx { "==>" } else { "" }); - for reg in 0..=num_regs { - eprint!("{:1}", if reg < live_regs[insn_idx] { "|" } else { "" }); - } - eprintln!(" [{:3}] {:?}", insn_idx, insn); - } - } - + pub(super) fn alloc_regs(mut self, regs: Vec<Reg>) -> Result<Assembler, CompileError> { // First, create the pool of registers. - let mut pool = RegisterPool::new(regs.clone()); + let mut pool = RegisterPool::new(regs.clone(), self.stack_base_idx); - // Mapping between VReg and allocated VReg for each VReg index. - // None if no register has been allocated for the VReg. - let mut reg_mapping: Vec<Option<Reg>> = vec![None; self.live_ranges.len()]; + // Mapping between VReg and register or stack slot for each VReg index. + // None if no register or stack slot has been allocated for the VReg. + let mut vreg_opnd: Vec<Option<Opnd>> = vec![None; self.live_ranges.len()]; // List of registers saved before a C call, paired with the VReg index. let mut saved_regs: Vec<(Reg, usize)> = vec![]; + // Remember the indexes of Insn::FrameSetup to update the stack size later + let mut frame_setup_idxs: Vec<usize> = vec![]; + // live_ranges is indexed by original `index` given by the iterator. + let mut asm = Assembler::new_with_asm(&self); let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); let mut iterator = self.insns.into_iter().enumerate().peekable(); - let mut asm = Assembler::new_with_label_names(take(&mut self.label_names), live_ranges.len()); while let Some((index, mut insn)) = iterator.next() { + // Remember the index of FrameSetup to bump slot_count when we know the max number of spilled VRegs. + if let Insn::FrameSetup { .. } = insn { + frame_setup_idxs.push(asm.insns.len()); + } + let before_ccall = match (&insn, iterator.peek().map(|(_, insn)| insn)) { (Insn::ParallelMov { .. }, Some(Insn::CCall { .. })) | (Insn::CCall { .. }, _) if !pool.is_empty() => { // If C_RET_REG is in use, move it to another register. // This must happen before last-use registers are deallocated. if let Some(vreg_idx) = pool.vreg_for(&C_RET_REG) { - let new_reg = pool.alloc_reg(vreg_idx).unwrap(); // TODO: support spill - asm.mov(Opnd::Reg(new_reg), C_RET_OPND); - pool.dealloc_reg(&C_RET_REG); - reg_mapping[vreg_idx] = Some(new_reg); + let new_opnd = pool.alloc_opnd(vreg_idx); + asm.mov(new_opnd, C_RET_OPND); + pool.dealloc_opnd(&Opnd::Reg(C_RET_REG)); + vreg_opnd[vreg_idx] = Some(new_opnd); } true @@ -1495,8 +1587,8 @@ impl Assembler // uses this operand. If it is, we can return the allocated // register to the pool. if live_ranges[idx].end() == index { - if let Some(reg) = reg_mapping[idx] { - pool.dealloc_reg(®); + if let Some(opnd) = vreg_opnd[idx] { + pool.dealloc_opnd(&opnd); } else { unreachable!("no register allocated for insn {:?}", insn); } @@ -1514,7 +1606,7 @@ impl Assembler // Save live registers for &(reg, _) in saved_regs.iter() { asm.cpush(Opnd::Reg(reg)); - pool.dealloc_reg(®); + pool.dealloc_opnd(&Opnd::Reg(reg)); } // On x86_64, maintain 16-byte stack alignment if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { @@ -1522,22 +1614,24 @@ impl Assembler } } - // If the output VReg of this instruction is used by another instruction, - // we need to allocate a register to it + // Allocate a register for the output operand if it exists let vreg_idx = match insn.out_opnd() { Some(Opnd::VReg { idx, .. }) => Some(*idx), _ => None, }; - if vreg_idx.is_some() && live_ranges[vreg_idx.unwrap()].end() != index { + if let Some(vreg_idx) = vreg_idx { + if live_ranges[vreg_idx].end() == index { + debug!("Allocating a register for VReg({}) at instruction index {} even though it does not live past this index", vreg_idx, index); + } // This is going to be the output operand that we will set on the // instruction. CCall and LiveReg need to use a specific register. let mut out_reg = match insn { Insn::CCall { .. } => { - Some(pool.take_reg(&C_RET_REG, vreg_idx.unwrap())) + Some(pool.take_reg(&C_RET_REG, vreg_idx)) } Insn::LiveReg { opnd, .. } => { let reg = opnd.unwrap_reg(); - Some(pool.take_reg(®, vreg_idx.unwrap())) + Some(pool.take_reg(®, vreg_idx)) } _ => None }; @@ -1552,8 +1646,8 @@ impl Assembler if let Some(Opnd::VReg{ idx, .. }) = opnd_iter.next() { if live_ranges[*idx].end() == index { - if let Some(reg) = reg_mapping[*idx] { - out_reg = Some(pool.take_reg(®, vreg_idx.unwrap())); + if let Some(Opnd::Reg(reg)) = vreg_opnd[*idx] { + out_reg = Some(pool.take_reg(®, vreg_idx)); } } } @@ -1561,22 +1655,7 @@ impl Assembler // Allocate a new register for this instruction if one is not // already allocated. - if out_reg.is_none() { - out_reg = match &insn { - _ => match pool.alloc_reg(vreg_idx.unwrap()) { - Some(reg) => Some(reg), - None => { - let mut insns = asm.insns; - insns.push(insn); - while let Some((_, insn)) = iterator.next() { - insns.push(insn); - } - dump_live_regs(insns, live_ranges, regs.len(), index); - unreachable!("Register spill not supported"); - } - } - }; - } + let out_opnd = out_reg.unwrap_or_else(|| pool.alloc_opnd(vreg_idx)); // Set the output operand on the instruction let out_num_bits = Opnd::match_num_bits_iter(insn.opnd_iter()); @@ -1585,9 +1664,9 @@ impl Assembler // output operand on this instruction because the live range // extends beyond the index of the instruction. let out = insn.out_opnd_mut().unwrap(); - let reg = out_reg.unwrap().with_num_bits(out_num_bits); - reg_mapping[out.vreg_idx()] = Some(reg); - *out = Opnd::Reg(reg); + let out_opnd = out_opnd.with_num_bits(out_num_bits); + vreg_opnd[out.vreg_idx()] = Some(out_opnd); + *out = out_opnd; } // Replace VReg and Param operands by their corresponding register @@ -1595,23 +1674,44 @@ impl Assembler while let Some(opnd) = opnd_iter.next() { match *opnd { Opnd::VReg { idx, num_bits } => { - *opnd = Opnd::Reg(reg_mapping[idx].unwrap()).with_num_bits(num_bits).unwrap(); + *opnd = vreg_opnd[idx].unwrap().with_num_bits(num_bits); }, Opnd::Mem(Mem { base: MemBase::VReg(idx), disp, num_bits }) => { - let base = MemBase::Reg(reg_mapping[idx].unwrap().reg_no); - *opnd = Opnd::Mem(Mem { base, disp, num_bits }); + *opnd = match vreg_opnd[idx].unwrap() { + Opnd::Reg(reg) => Opnd::Mem(Mem { base: MemBase::Reg(reg.reg_no), disp, num_bits }), + // If the base is spilled, lower it to MemBase::Stack, which scratch_split will lower to MemBase::Reg. + Opnd::Mem(mem) => Opnd::Mem(Mem { base: pool.stack_state.mem_to_stack_membase(mem), disp, num_bits }), + _ => unreachable!(), + } } _ => {}, } } + // If we have an output that dies at its definition (it is unused), free up the + // register + if let Some(idx) = vreg_idx { + if live_ranges[idx].end() == index { + if let Some(opnd) = vreg_opnd[idx] { + pool.dealloc_opnd(&opnd); + } else { + unreachable!("no register allocated for insn {:?}", insn); + } + } + } + // Push instruction(s) let is_ccall = matches!(insn, Insn::CCall { .. }); match insn { Insn::ParallelMov { moves } => { - // Now that register allocation is done, it's ready to resolve parallel moves. - for (reg, opnd) in Self::resolve_parallel_moves(&moves) { - asm.load_into(Opnd::Reg(reg), opnd); + // For trampolines that use scratch registers, attempt to lower ParallelMov without scratch_reg. + if let Some(moves) = Self::resolve_parallel_moves(&moves, None) { + for (dst, src) in moves { + asm.mov(dst, src); + } + } else { + // If it needs a scratch_reg, leave it to *_split_with_scratch_regs to handle it. + asm.push_insn(Insn::ParallelMov { moves }); } } Insn::CCall { opnds, fptr, start_marker, end_marker, out } => { @@ -1624,6 +1724,9 @@ impl Assembler asm.push_insn(Insn::PosMarker(end_marker)); } } + Insn::Mov { src, dest } | Insn::LoadInto { dest, opnd: src } if src == dest => { + // Remove no-op move now that VReg are resolved to physical Reg + } _ => asm.push_insn(insn), } @@ -1631,7 +1734,7 @@ impl Assembler if is_ccall { // On x86_64, maintain 16-byte stack alignment if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { - asm.cpop_into(Opnd::Reg(saved_regs.last().unwrap().0.clone())); + asm.cpop_into(Opnd::Reg(saved_regs.last().unwrap().0)); } // Restore saved registers for &(reg, vreg_idx) in saved_regs.iter().rev() { @@ -1642,21 +1745,35 @@ impl Assembler } } + // Extend the stack space for spilled operands + for frame_setup_idx in frame_setup_idxs { + match &mut asm.insns[frame_setup_idx] { + Insn::FrameSetup { slot_count, .. } => { + *slot_count += pool.stack_state.stack_size; + } + _ => unreachable!(), + } + } + assert!(pool.is_empty(), "Expected all registers to be returned to the pool"); - asm + Ok(asm) } /// Compile the instructions down to machine code. /// Can fail due to lack of code memory and inopportune code placement, among other reasons. - #[must_use] - pub fn compile(mut self, cb: &mut CodeBlock) -> Option<(CodePtr, Vec<u32>)> - { - self.compile_side_exits(cb)?; - + pub fn compile(self, cb: &mut CodeBlock) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { #[cfg(feature = "disasm")] let start_addr = cb.get_write_ptr(); let alloc_regs = Self::get_alloc_regs(); - let ret = self.compile_with_regs(cb, alloc_regs); + let had_dropped_bytes = cb.has_dropped_bytes(); + let ret = self.compile_with_regs(cb, alloc_regs).inspect_err(|err| { + // If we use too much memory to compile the Assembler, it would set cb.dropped_bytes = true. + // To avoid failing future compilation by cb.has_dropped_bytes(), attempt to reset dropped_bytes with + // the current zjit_alloc_bytes() which may be decreased after self is dropped in compile_with_regs(). + if *err == CompileError::OutOfMemory && !had_dropped_bytes { + cb.update_dropped_bytes(); + } + }); #[cfg(feature = "disasm")] if get_option!(dump_disasm) { @@ -1667,49 +1784,216 @@ impl Assembler ret } + /// Compile with a limited number of registers. Used only for unit tests. + #[cfg(test)] + pub fn compile_with_num_regs(self, cb: &mut CodeBlock, num_regs: usize) -> (CodePtr, Vec<CodePtr>) { + let mut alloc_regs = Self::get_alloc_regs(); + let alloc_regs = alloc_regs.drain(0..num_regs).collect(); + self.compile_with_regs(cb, alloc_regs).unwrap() + } + /// Compile Target::SideExit and convert it into Target::CodePtr for all instructions - #[must_use] - pub fn compile_side_exits(&mut self, cb: &mut CodeBlock) -> Option<()> { - for insn in self.insns.iter_mut() { - if let Some(target) = insn.target_mut() { - if let Target::SideExit(state) = target { - let side_exit_ptr = cb.get_write_ptr(); - let mut asm = Assembler::new(); - asm_comment!(asm, "side exit: {state}"); - asm.ccall(Self::rb_zjit_side_exit as *const u8, vec![]); - asm.compile(cb)?; - *target = Target::SideExitPtr(side_exit_ptr); + pub fn compile_exits(&mut self) { + /// Compile the main side-exit code. This function takes only SideExit so + /// that it can be safely deduplicated by using SideExit as a dedup key. + fn compile_exit(asm: &mut Assembler, exit: &SideExit) { + let SideExit { pc, stack, locals } = exit; + + asm_comment!(asm, "save cfp->pc"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), *pc); + + asm_comment!(asm, "save cfp->sp"); + asm.lea_into(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); + + if !stack.is_empty() { + asm_comment!(asm, "write stack slots: {}", join_opnds(&stack, ", ")); + for (idx, &opnd) in stack.iter().enumerate() { + asm.store(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), opnd); } } + + if !locals.is_empty() { + asm_comment!(asm, "write locals: {}", join_opnds(&locals, ", ")); + for (idx, &opnd) in locals.iter().enumerate() { + asm.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); + } + } + + asm_comment!(asm, "exit to the interpreter"); + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(Opnd::UImm(Qundef.as_u64())); } - Some(()) - } - #[unsafe(no_mangle)] - extern "C" fn rb_zjit_side_exit() { - unimplemented!("side exits are not implemented yet"); - } + fn join_opnds(opnds: &Vec<Opnd>, delimiter: &str) -> String { + opnds.iter().map(|opnd| format!("{opnd}")).collect::<Vec<_>>().join(delimiter) + } - /* - /// Compile with a limited number of registers. Used only for unit tests. - #[cfg(test)] - pub fn compile_with_num_regs(self, cb: &mut CodeBlock, num_regs: usize) -> (CodePtr, Vec<u32>) - { - let mut alloc_regs = Self::get_alloc_regs(); - let alloc_regs = alloc_regs.drain(0..num_regs).collect(); - self.compile_with_regs(cb, None, alloc_regs).unwrap() - } + // Extract targets first so that we can update instructions while referencing part of them. + let mut targets = HashMap::new(); + for (idx, insn) in self.insns.iter().enumerate() { + if let Some(target @ Target::SideExit { .. }) = insn.target() { + targets.insert(idx, target.clone()); + } + } + + // Map from SideExit to compiled Label. This table is used to deduplicate side exit code. + let mut compiled_exits: HashMap<SideExit, Label> = HashMap::new(); + + for (idx, target) in targets { + // Compile a side exit. Note that this is past the split pass and alloc_regs(), + // so you can't use an instruction that returns a VReg. + if let Target::SideExit { exit: exit @ SideExit { pc, .. }, reason } = target { + // Only record the exit if `trace_side_exits` is defined and the counter is either the one specified + let should_record_exit = get_option!(trace_side_exits).map(|trace| match trace { + TraceExits::All => true, + TraceExits::Counter(counter) if counter == side_exit_counter(reason) => true, + _ => false, + }).unwrap_or(false); + + // If enabled, instrument exits first, and then jump to a shared exit. + let counted_exit = if get_option!(stats) || should_record_exit { + let counted_exit = self.new_label("counted_exit"); + self.write_label(counted_exit.clone()); + asm_comment!(self, "Counted Exit: {reason}"); + + if get_option!(stats) { + asm_comment!(self, "increment a side exit counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr(reason)), 1.into()); + + if let SideExitReason::UnhandledYARVInsn(opcode) = reason { + asm_comment!(self, "increment an unhandled YARV insn counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr_for_opcode(opcode)), 1.into()); + } + } - /// Return true if the next ccall() is expected to be leaf. - pub fn get_leaf_ccall(&mut self) -> bool { - self.leaf_ccall + if should_record_exit { + // Preserve caller-saved registers that may be used in the shared exit. + self.cpush_all(); + asm_ccall!(self, rb_zjit_record_exit_stack, pc); + self.cpop_all(); + } + + // If the side exit has already been compiled, jump to it. + // Otherwise, let it fall through and compile the exit next. + if let Some(&exit_label) = compiled_exits.get(&exit) { + self.jmp(Target::Label(exit_label)); + } + Some(counted_exit) + } else { + None + }; + + // Compile the shared side exit if not compiled yet + let compiled_exit = if let Some(&compiled_exit) = compiled_exits.get(&exit) { + Target::Label(compiled_exit) + } else { + let new_exit = self.new_label("side_exit"); + self.write_label(new_exit.clone()); + asm_comment!(self, "Exit: {pc}"); + compile_exit(self, &exit); + compiled_exits.insert(exit, new_exit.unwrap_label()); + new_exit + }; + + *self.insns[idx].target_mut().unwrap() = counted_exit.unwrap_or(compiled_exit); + } + } } +} + +/// Return a result of fmt::Display for Assembler without escape sequence +pub fn lir_string(asm: &Assembler) -> String { + use crate::ttycolors::TTY_TERMINAL_COLOR; + format!("{asm}").replace(TTY_TERMINAL_COLOR.bold_begin, "").replace(TTY_TERMINAL_COLOR.bold_end, "") +} + +impl fmt::Display for Assembler { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Count the number of duplicated label names to disambiguate them if needed + let mut label_counts: HashMap<&String, usize> = HashMap::new(); + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + for label_name in self.label_names.iter() { + let counter = label_counts.entry(label_name).or_insert(0); + *counter += 1; + } + + /// Return a label name String. Suffix "_{label_idx}" if the label name is used multiple times. + fn label_name(asm: &Assembler, label_idx: usize, label_counts: &HashMap<&String, usize>) -> String { + let label_name = &asm.label_names[label_idx]; + let label_count = label_counts.get(&label_name).unwrap_or(&0); + if *label_count > 1 { + format!("{label_name}_{label_idx}") + } else { + label_name.to_string() + } + } + + for insn in self.insns.iter() { + match insn { + Insn::Comment(comment) => { + writeln!(f, " {bold_begin}# {comment}{bold_end}")?; + } + Insn::Label(target) => { + let &Target::Label(Label(label_idx)) = target else { + panic!("unexpected target for Insn::Label: {target:?}"); + }; + writeln!(f, " {}:", label_name(self, label_idx, &label_counts))?; + } + _ => { + write!(f, " ")?; + + // Print output operand if any + if let Some(out) = insn.out_opnd() { + write!(f, "{out} = ")?; + } + + // Print the instruction name + write!(f, "{}", insn.op())?; + + // Show slot_count for FrameSetup + if let Insn::FrameSetup { slot_count, preserved } = insn { + write!(f, " {slot_count}")?; + if !preserved.is_empty() { + write!(f, ",")?; + } + } + + // Print target + if let Some(target) = insn.target() { + match target { + Target::CodePtr(code_ptr) => write!(f, " {code_ptr:?}")?, + Target::Label(Label(label_idx)) => write!(f, " {}", label_name(self, *label_idx, &label_counts))?, + Target::SideExit { reason, .. } => write!(f, " Exit({reason})")?, + } + } + + // Print list of operands + if let Some(Target::SideExit { .. }) = insn.target() { + // If the instruction has a SideExit, avoid using opnd_iter(), which has stack/locals. + // Here, only handle instructions that have both Opnd and Target. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } + } else if let Insn::ParallelMov { moves } = insn { + // Print operands with a special syntax for ParallelMov + moves.iter().try_fold(" ", |prefix, (dst, src)| write!(f, "{prefix}{dst} <- {src}").and(Ok(", ")))?; + } else if insn.opnd_iter().count() > 0 { + insn.opnd_iter().try_fold(" ", |prefix, opnd| write!(f, "{prefix}{opnd}").and(Ok(", ")))?; + } - /// Assert that the next ccall() is going to be leaf. - pub fn expect_leaf_ccall(&mut self) { - self.leaf_ccall = true; + write!(f, "\n")?; + } + } + } + Ok(()) } - */ } impl fmt::Debug for Assembler { @@ -1724,6 +2008,44 @@ impl fmt::Debug for Assembler { } } +pub struct InsnIter { + old_insns_iter: std::vec::IntoIter<Insn>, + peeked: Option<(usize, Insn)>, + index: usize, +} + +impl InsnIter { + // We're implementing our own peek() because we don't want peek to + // cross basic blocks as we're iterating. + pub fn peek(&mut self) -> Option<&(usize, Insn)> { + // If we don't have a peeked value, get one + if self.peeked.is_none() { + let insn = self.old_insns_iter.next()?; + let idx = self.index; + self.index += 1; + self.peeked = Some((idx, insn)); + } + // Return a reference to the peeked value + self.peeked.as_ref() + } + + // Get the next instruction. Right now we're passing the "new" assembler + // (the assembler we're copying in to) as a parameter. Once we've + // introduced basic blocks to LIR, we'll use the to set the correct BB + // on the new assembler, but for now it is unused. + pub fn next(&mut self, _new_asm: &mut Assembler) -> Option<(usize, Insn)> { + // If we have a peeked value, return it + if let Some(item) = self.peeked.take() { + return Some(item); + } + // Otherwise get the next from underlying iterator + let insn = self.old_insns_iter.next()?; + let idx = self.index; + self.index += 1; + Some((idx, insn)) + } +} + impl Assembler { #[must_use] pub fn add(&mut self, left: Opnd, right: Opnd) -> Opnd { @@ -1732,6 +2054,11 @@ impl Assembler { out } + pub fn add_into(&mut self, left: Opnd, right: Opnd) { + assert!(matches!(left, Opnd::Reg(_)), "Destination of add_into must be Opnd::Reg, but got: {left:?}"); + self.push_insn(Insn::Add { left, right, out: left }); + } + #[must_use] pub fn and(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); @@ -1750,8 +2077,19 @@ impl Assembler { /// Call a C function without PosMarkers pub fn ccall(&mut self, fptr: *const u8, opnds: Vec<Opnd>) -> Opnd { + let canary_opnd = self.set_stack_canary(); let out = self.new_vreg(Opnd::match_num_bits(&opnds)); + let fptr = Opnd::const_ptr(fptr); self.push_insn(Insn::CCall { fptr, opnds, start_marker: None, end_marker: None, out }); + self.clear_stack_canary(canary_opnd); + out + } + + /// Call a C function stored in a register + pub fn ccall_reg(&mut self, fptr: Opnd, num_bits: u8) -> Opnd { + assert!(matches!(fptr, Opnd::Reg(_)), "ccall_reg must be called with Opnd::Reg: {fptr:?}"); + let out = self.new_vreg(num_bits); + self.push_insn(Insn::CCall { fptr, opnds: vec![], start_marker: None, end_marker: None, out }); out } @@ -1766,39 +2104,25 @@ impl Assembler { ) -> Opnd { let out = self.new_vreg(Opnd::match_num_bits(&opnds)); self.push_insn(Insn::CCall { - fptr, + fptr: Opnd::const_ptr(fptr), opnds, - start_marker: Some(Box::new(start_marker)), - end_marker: Some(Box::new(end_marker)), + start_marker: Some(Rc::new(start_marker)), + end_marker: Some(Rc::new(end_marker)), out, }); out } - /* - /// Let vm_check_canary() assert the leafness of this ccall if leaf_ccall is set - fn set_stack_canary(&mut self, opnds: &Vec<Opnd>) -> Option<Opnd> { - // Use the slot right above the stack top for verifying leafness. - let canary_opnd = self.stack_opnd(-1); - - // If the slot is already used, which is a valid optimization to avoid spills, - // give up the verification. - let canary_opnd = if cfg!(feature = "runtime_checks") && self.leaf_ccall && opnds.iter().all(|opnd| - opnd.get_reg_opnd() != canary_opnd.get_reg_opnd() - ) { - asm_comment!(self, "set stack canary"); - self.mov(canary_opnd, vm_stack_canary().into()); - Some(canary_opnd) - } else { - None - }; - - // Avoid carrying the flag to the next instruction whether we verified it or not. - self.leaf_ccall = false; - - canary_opnd + pub fn count_call_to(&mut self, fn_name: &str) { + // We emit ccalls while initializing the JIT. Unfortunately, we skip those because + // otherwise we have no counter pointers to read. + if crate::state::ZJITState::has_instance() && get_option!(stats) { + let ccall_counter_pointers = crate::state::ZJITState::get_ccall_counter_pointers(); + let counter_ptr = ccall_counter_pointers.entry(fn_name.to_string()).or_insert_with(|| Box::new(0)); + let counter_ptr: &mut u64 = counter_ptr.as_mut(); + self.incr_counter(Opnd::const_ptr(counter_ptr), 1.into()); + } } - */ pub fn cmp(&mut self, left: Opnd, right: Opnd) { self.push_insn(Insn::Cmp { left, right }); @@ -1813,13 +2137,10 @@ impl Assembler { pub fn cpop_all(&mut self) { self.push_insn(Insn::CPopAll); - - // Re-enable ccall's RegMappings assertion disabled by cpush_all. - // cpush_all + cpop_all preserve all stack temp registers, so it's safe. - //self.set_reg_mapping(self.ctx.get_reg_mapping()); } pub fn cpop_into(&mut self, opnd: Opnd) { + assert!(matches!(opnd, Opnd::Reg(_)), "Destination of cpop_into must be a register, got: {opnd:?}"); self.push_insn(Insn::CPopInto(opnd)); } @@ -1829,12 +2150,6 @@ impl Assembler { pub fn cpush_all(&mut self) { self.push_insn(Insn::CPushAll); - - // Mark all temps as not being in registers. - // Temps will be marked back as being in registers by cpop_all. - // We assume that cpush_all + cpop_all are used for C functions in utils.rs - // that don't require spill_regs for GC. - //self.set_reg_mapping(RegMapping::default()); } pub fn cret(&mut self, opnd: Opnd) { @@ -1897,12 +2212,15 @@ impl Assembler { out } - pub fn frame_setup(&mut self) { - self.push_insn(Insn::FrameSetup); + pub fn frame_setup(&mut self, preserved_regs: &'static [Opnd]) { + let slot_count = self.stack_base_idx; + self.push_insn(Insn::FrameSetup { preserved: preserved_regs, slot_count }); } - pub fn frame_teardown(&mut self) { - self.push_insn(Insn::FrameTeardown); + /// The inverse of [Self::frame_setup] used before return. `reserve_bytes` + /// not necessary since we use a base pointer register. + pub fn frame_teardown(&mut self, preserved_regs: &'static [Opnd]) { + self.push_insn(Insn::FrameTeardown { preserved: preserved_regs }); } pub fn incr_counter(&mut self, mem: Opnd, value: Opnd) { @@ -1970,6 +2288,11 @@ impl Assembler { out } + pub fn lea_into(&mut self, out: Opnd, opnd: Opnd) { + assert!(matches!(out, Opnd::Reg(_) | Opnd::Mem(_)), "Destination of lea_into must be a register or memory, got: {out:?}"); + self.push_insn(Insn::Lea { opnd, out }); + } + #[must_use] pub fn lea_jump_target(&mut self, target: Target) -> Opnd { let out = self.new_vreg(Opnd::DEFAULT_NUM_BITS); @@ -1992,6 +2315,7 @@ impl Assembler { } pub fn load_into(&mut self, dest: Opnd, opnd: Opnd) { + assert!(matches!(dest, Opnd::Reg(_)), "Destination of load_into must be a register, got: {dest:?}"); match (dest, opnd) { (Opnd::Reg(dest), Opnd::Reg(opnd)) if dest == opnd => {}, // skip if noop _ => self.push_insn(Insn::LoadInto { dest, opnd }), @@ -2012,11 +2336,12 @@ impl Assembler { out } - pub fn parallel_mov(&mut self, moves: Vec<(Reg, Opnd)>) { + pub fn parallel_mov(&mut self, moves: Vec<(Opnd, Opnd)>) { self.push_insn(Insn::ParallelMov { moves }); } pub fn mov(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of mov must not be Opnd::VReg, got: {dest:?}"); self.push_insn(Insn::Mov { dest, src }); } @@ -2034,13 +2359,16 @@ impl Assembler { out } - pub fn pad_inval_patch(&mut self) { - self.push_insn(Insn::PadInvalPatch); + pub fn patch_point(&mut self, target: Target, invariant: Invariant, version: IseqVersionRef) { + self.push_insn(Insn::PatchPoint { target, invariant, version }); + } + + pub fn pad_patch_point(&mut self) { + self.push_insn(Insn::PadPatchPoint); } - //pub fn pos_marker<F: FnMut(CodePtr)>(&mut self, marker_fn: F) pub fn pos_marker(&mut self, marker_fn: impl Fn(CodePtr, &CodeBlock) + 'static) { - self.push_insn(Insn::PosMarker(Box::new(marker_fn))); + self.push_insn(Insn::PosMarker(Rc::new(marker_fn))); } #[must_use] @@ -2051,6 +2379,7 @@ impl Assembler { } pub fn store(&mut self, dest: Opnd, src: Opnd) { + assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of store must not be Opnd::VReg, got: {dest:?}"); self.push_insn(Insn::Store { dest, src }); } @@ -2061,6 +2390,11 @@ impl Assembler { out } + pub fn sub_into(&mut self, left: Opnd, right: Opnd) { + assert!(matches!(left, Opnd::Reg(_)), "Destination of sub_into must be Opnd::Reg, but got: {left:?}"); + self.push_insn(Insn::Sub { left, right, out: left }); + } + #[must_use] pub fn mul(&mut self, left: Opnd, right: Opnd) -> Opnd { let out = self.new_vreg(Opnd::match_num_bits(&[left, right])); @@ -2080,18 +2414,6 @@ impl Assembler { out } - /* - /// Verify the leafness of the given block - pub fn with_leaf_ccall<F, R>(&mut self, mut block: F) -> R - where F: FnMut(&mut Self) -> R { - let old_leaf_ccall = self.leaf_ccall; - self.leaf_ccall = true; - let ret = block(self); - self.leaf_ccall = old_leaf_ccall; - ret - } - */ - /// Add a label at the current position pub fn write_label(&mut self, target: Target) { assert!(target.unwrap_label().0 < self.label_names.len()); @@ -2110,17 +2432,121 @@ impl Assembler { /// when not dumping disassembly. macro_rules! asm_comment { ($asm:expr, $($fmt:tt)*) => { - if $crate::options::get_option!(dump_disasm) { + // If --zjit-dump-disasm or --zjit-dump-lir is given, enrich them with comments. + // Also allow --zjit-debug on dev builds to enable comments since dev builds dump LIR on panic. + let enable_comment = $crate::options::get_option!(dump_disasm) || + $crate::options::get_option!(dump_lir).is_some() || + (cfg!(debug_assertions) && $crate::options::get_option!(debug)); + if enable_comment { $asm.push_insn(crate::backend::lir::Insn::Comment(format!($($fmt)*))); } }; } pub(crate) use asm_comment; +/// Convenience macro over [`Assembler::ccall`] that also adds a comment with the function name. +macro_rules! asm_ccall { + [$asm: ident, $fn_name:ident, $($args:expr),* ] => {{ + $crate::backend::lir::asm_comment!($asm, concat!("call ", stringify!($fn_name))); + $asm.count_call_to(stringify!($fn_name)); + $asm.ccall($fn_name as *const u8, vec![$($args),*]) + }}; +} +pub(crate) use asm_ccall; + +// Allow moving Assembler to panic hooks. Since we take the VM lock on compilation, +// no other threads should reference the same Assembler instance. +unsafe impl Send for Insn {} +unsafe impl Sync for Insn {} + +/// Dump Assembler with insn_idx on panic. Restore the original panic hook on drop. +pub struct AssemblerPanicHook { + /// Original panic hook before AssemblerPanicHook is installed. + prev_hook: Box<dyn Fn(&panic::PanicHookInfo<'_>) + Sync + Send + 'static>, +} + +impl AssemblerPanicHook { + /// Maximum number of lines [`Self::dump_asm`] is allowed to dump by default. + /// When --zjit-dump-lir is given, this limit is ignored. + const MAX_DUMP_LINES: usize = 10; + + /// Install a panic hook to dump Assembler with insn_idx on dev builds. + /// This returns shared references to the previous hook and insn_idx. + /// It takes insn_idx as an argument so that you can manually use it + /// on non-emit passes that keep mutating the Assembler to be dumped. + pub fn new(asm: &Assembler, insn_idx: usize) -> (Option<Arc<Self>>, Option<Arc<Mutex<usize>>>) { + if cfg!(debug_assertions) { + // Wrap prev_hook with Arc to share it among the new hook and Self to be dropped. + let prev_hook = panic::take_hook(); + let panic_hook_ref = Arc::new(Self { prev_hook }); + let weak_hook = Arc::downgrade(&panic_hook_ref); + + // Wrap insn_idx with Arc to share it among the new hook and the caller mutating it. + let insn_idx = Arc::new(Mutex::new(insn_idx)); + let insn_idx_ref = insn_idx.clone(); + + // Install a new hook to dump Assembler with insn_idx + let asm = asm.clone(); + panic::set_hook(Box::new(move |panic_info| { + if let Some(panic_hook) = weak_hook.upgrade() { + if let Ok(insn_idx) = insn_idx_ref.lock() { + // Dump Assembler, highlighting the insn_idx line + Self::dump_asm(&asm, *insn_idx); + } + + // Call the previous panic hook + (panic_hook.prev_hook)(panic_info); + } + })); + + (Some(panic_hook_ref), Some(insn_idx)) + } else { + (None, None) + } + } + + /// Dump Assembler, highlighting the insn_idx line + fn dump_asm(asm: &Assembler, insn_idx: usize) { + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + let lir_string = lir_string(asm); + let lines: Vec<&str> = lir_string.split('\n').collect(); + + // By default, dump only MAX_DUMP_LINES lines. + // Ignore it if --zjit-dump-lir is given. + let (min_idx, max_idx) = if get_option!(dump_lir).is_some() { + (0, lines.len()) + } else { + (insn_idx.saturating_sub(Self::MAX_DUMP_LINES / 2), insn_idx.saturating_add(Self::MAX_DUMP_LINES / 2)) + }; + + eprintln!("Failed to compile LIR at insn_idx={insn_idx}:"); + for (idx, line) in lines.iter().enumerate().filter(|(idx, _)| (min_idx..=max_idx).contains(idx)) { + if idx == insn_idx && line.starts_with(" ") { + eprintln!("{bold_begin}=>{}{bold_end}", &line[" ".len()..]); + } else { + eprintln!("{line}"); + } + } + } +} + +impl Drop for AssemblerPanicHook { + fn drop(&mut self) { + // Restore the original hook + panic::set_hook(std::mem::replace(&mut self.prev_hook, Box::new(|_| {}))); + } +} + #[cfg(test)] mod tests { use super::*; + fn scratch_reg() -> Opnd { + Assembler::new_with_scratch_reg().1 + } + #[test] fn test_opnd_iter() { let insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; @@ -2129,7 +2555,7 @@ mod tests { assert!(matches!(opnd_iter.next(), Some(Opnd::None))); assert!(matches!(opnd_iter.next(), Some(Opnd::None))); - assert!(matches!(opnd_iter.next(), None)); + assert!(opnd_iter.next().is_none()); } #[test] @@ -2140,7 +2566,109 @@ mod tests { assert!(matches!(opnd_iter.next(), Some(Opnd::None))); assert!(matches!(opnd_iter.next(), Some(Opnd::None))); - assert!(matches!(opnd_iter.next(), None)); + assert!(opnd_iter.next().is_none()); } -} + #[test] + #[should_panic] + fn load_into_memory_is_invalid() { + let mut asm = Assembler::new(); + let mem = Opnd::mem(64, SP, 0); + asm.load_into(mem, mem); + } + + #[test] + fn test_resolve_parallel_moves_reorder_registers() { + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, Some(vec![ + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + fn test_resolve_parallel_moves_give_up_register_cycle() { + // If scratch_opnd is not given, it cannot break cycles. + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, None); + } + + #[test] + fn test_resolve_parallel_moves_break_register_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_memory_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_register_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_reorder_memory_destination() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_register() { + Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[0], CFP), + ], Some(scratch_reg())); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_memory() { + Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg())); + } +} diff --git a/zjit/src/backend/mod.rs b/zjit/src/backend/mod.rs index 4922421f18..635acbf60c 100644 --- a/zjit/src/backend/mod.rs +++ b/zjit/src/backend/mod.rs @@ -1,3 +1,5 @@ +//! A multi-platform assembler generation backend. + #[cfg(target_arch = "x86_64")] pub mod x86_64; @@ -10,4 +12,7 @@ pub use x86_64 as current; #[cfg(target_arch = "aarch64")] pub use arm64 as current; +#[cfg(test)] +mod tests; + pub mod lir; diff --git a/zjit/src/backend/tests.rs b/zjit/src/backend/tests.rs index b3918a05f9..ece6f8605f 100644 --- a/zjit/src/backend/tests.rs +++ b/zjit/src/backend/tests.rs @@ -1,20 +1,20 @@ -/* -#![cfg(test)] use crate::asm::CodeBlock; -use crate::backend::*; +use crate::backend::lir::*; use crate::cruby::*; -use crate::utils::c_callable; +use crate::codegen::c_callable; +use crate::options::rb_zjit_prepare_options; #[test] fn test_add() { - let mut asm = Assembler::new(0); + let mut asm = Assembler::new(); let out = asm.add(SP, Opnd::UImm(1)); let _ = asm.add(out, Opnd::UImm(2)); } #[test] fn test_alloc_regs() { - let mut asm = Assembler::new(0); + rb_zjit_prepare_options(); // for asm.alloc_regs + let mut asm = Assembler::new(); // Get the first output that we're going to reuse later. let out1 = asm.add(EC, Opnd::UImm(1)); @@ -37,7 +37,7 @@ fn test_alloc_regs() { let _ = asm.add(out3, Opnd::UImm(6)); // Here we're going to allocate the registers. - let result = asm.alloc_regs(Assembler::get_alloc_regs()); + let result = asm.alloc_regs(Assembler::get_alloc_regs()).unwrap(); // Now we're going to verify that the out field has been appropriately // updated for each of the instructions that needs it. @@ -62,10 +62,8 @@ fn test_alloc_regs() { } fn setup_asm() -> (Assembler, CodeBlock) { - return ( - Assembler::new(0), - CodeBlock::new_dummy(1024) - ); + rb_zjit_prepare_options(); // for get_option! on asm.compile + (Assembler::new(), CodeBlock::new_dummy()) } // Test full codegen pipeline @@ -164,11 +162,10 @@ fn test_base_insn_out() ); // Load the pointer into a register - let ptr_reg = asm.load(Opnd::const_ptr(4351776248 as *const u8)); - let counter_opnd = Opnd::mem(64, ptr_reg, 0); + let ptr_opnd = Opnd::const_ptr(4351776248 as *const u8); // Increment and store the updated value - asm.incr_counter(counter_opnd, 1.into()); + asm.incr_counter(ptr_opnd, 1.into()); asm.compile_with_num_regs(&mut cb, 2); } @@ -190,17 +187,16 @@ fn test_c_call() // Make sure that the call's return value is usable asm.mov(Opnd::mem(64, SP, 0), ret_val); - asm.compile_with_num_regs(&mut cb, 1); + asm.compile(&mut cb).unwrap(); } #[test] fn test_alloc_ccall_regs() { - let mut asm = Assembler::new(0); - let out1 = asm.ccall(0 as *const u8, vec![]); - let out2 = asm.ccall(0 as *const u8, vec![out1]); + let (mut asm, mut cb) = setup_asm(); + let out1 = asm.ccall(std::ptr::null::<u8>(), vec![]); + let out2 = asm.ccall(std::ptr::null::<u8>(), vec![out1]); asm.mov(EC, out2); - let mut cb = CodeBlock::new_dummy(1024); - asm.compile_with_regs(&mut cb, None, Assembler::get_alloc_regs()); + asm.compile_with_regs(&mut cb, Assembler::get_alloc_regs()).unwrap(); } #[test] @@ -221,7 +217,7 @@ fn test_jcc_label() let label = asm.new_label("foo"); asm.cmp(EC, EC); - asm.je(label); + asm.je(label.clone()); asm.write_label(label); asm.compile_with_num_regs(&mut cb, 1); @@ -233,9 +229,9 @@ fn test_jcc_ptr() let (mut asm, mut cb) = setup_asm(); let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); - let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK)); + let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK as i32)); asm.test( - Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG), + Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32), not_mask, ); asm.jnz(side_exit); @@ -283,30 +279,10 @@ fn test_bake_string() { } #[test] -fn test_draining_iterator() { - let mut asm = Assembler::new(0); - - let _ = asm.load(Opnd::None); - asm.store(Opnd::None, Opnd::None); - let _ = asm.add(Opnd::None, Opnd::None); - - let mut iter = asm.into_draining_iter(); - - while let Some((index, insn)) = iter.next_unmapped() { - match index { - 0 => assert!(matches!(insn, Insn::Load { .. })), - 1 => assert!(matches!(insn, Insn::Store { .. })), - 2 => assert!(matches!(insn, Insn::Add { .. })), - _ => panic!("Unexpected instruction index"), - }; - } -} - -#[test] fn test_cmp_8_bit() { let (mut asm, mut cb) = setup_asm(); let reg = Assembler::get_alloc_regs()[0]; - asm.cmp(Opnd::Reg(reg).with_num_bits(8).unwrap(), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.cmp(Opnd::Reg(reg).with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); asm.compile_with_num_regs(&mut cb, 1); } @@ -315,7 +291,8 @@ fn test_cmp_8_bit() { fn test_no_pos_marker_callback_when_compile_fails() { // When compilation fails (e.g. when out of memory), the code written out is malformed. // We don't want to invoke the pos_marker callbacks with positions of malformed code. - let mut asm = Assembler::new(0); + let mut asm = Assembler::new(); + rb_zjit_prepare_options(); // for asm.compile // Markers around code to exhaust memory limit let fail_if_called = |_code_ptr, _cb: &_| panic!("pos_marker callback should not be called"); @@ -325,8 +302,6 @@ fn test_no_pos_marker_callback_when_compile_fails() { asm.store(Opnd::mem(64, SP, 8), sum); asm.pos_marker(fail_if_called); - let cb = &mut CodeBlock::new_dummy(8); - assert!(asm.compile(cb, None).is_none(), "should fail due to tiny size limit"); + let cb = &mut CodeBlock::new_dummy_sized(8); + assert!(asm.compile(cb).is_err(), "should fail due to tiny size limit"); } - -*/ diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index f11b07c1b7..9f780617cc 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1,22 +1,30 @@ -use std::mem::take; +use std::mem::{self, take}; use crate::asm::*; use crate::asm::x86_64::*; +use crate::codegen::split_patch_point; +use crate::stats::CompileError; use crate::virtualmem::CodePtr; use crate::cruby::*; use crate::backend::lir::*; use crate::cast::*; +use crate::options::asm_dump; // Use the x86 register type for this platform pub type Reg = X86Reg; +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit GP register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_type: RegType::GP, reg_no } +} + // Callee-saved registers -pub const _CFP: Opnd = Opnd::Reg(R13_REG); -pub const _EC: Opnd = Opnd::Reg(R12_REG); -pub const _SP: Opnd = Opnd::Reg(RBX_REG); +pub const CFP: Opnd = Opnd::Reg(R13_REG); +pub const EC: Opnd = Opnd::Reg(R12_REG); +pub const SP: Opnd = Opnd::Reg(RBX_REG); // C argument registers on this platform -pub const _C_ARG_OPNDS: [Opnd; 6] = [ +pub const C_ARG_OPNDS: [Opnd; 6] = [ Opnd::Reg(RDI_REG), Opnd::Reg(RSI_REG), Opnd::Reg(RDX_REG), @@ -27,7 +35,9 @@ pub const _C_ARG_OPNDS: [Opnd; 6] = [ // C return value register on this platform pub const C_RET_REG: Reg = RAX_REG; -pub const _C_RET_OPND: Opnd = Opnd::Reg(RAX_REG); +pub const C_RET_OPND: Opnd = Opnd::Reg(RAX_REG); +pub const NATIVE_STACK_PTR: Opnd = Opnd::Reg(RSP_REG); +pub const NATIVE_BASE_PTR: Opnd = Opnd::Reg(RBP_REG); impl CodeBlock { // The number of bytes that are generated by jmp_ptr @@ -66,7 +76,7 @@ impl From<Opnd> for X86Opnd { "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." ), - _ => panic!("unsupported x86 operand type") + _ => panic!("unsupported x86 operand type: {opnd:?}") } } } @@ -80,24 +90,32 @@ impl From<&Opnd> for X86Opnd { /// List of registers that can be used for register allocation. /// This has the same number of registers for x86_64 and arm64. -/// SCRATCH_REG is excluded. -pub const ALLOC_REGS: &'static [Reg] = &[ +/// SCRATCH0_OPND is excluded. +pub const ALLOC_REGS: &[Reg] = &[ RDI_REG, RSI_REG, RDX_REG, RCX_REG, R8_REG, R9_REG, - R10_REG, RAX_REG, ]; -impl Assembler -{ - // A special scratch register for intermediate processing. - // This register is caller-saved (so we don't have to save it before using it) - pub const SCRATCH_REG: Reg = R11_REG; - const SCRATCH0: X86Opnd = X86Opnd::Reg(Assembler::SCRATCH_REG); +/// Special scratch register for intermediate processing. It should be used only by +/// [`Assembler::x86_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(R11_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(R10_REG); + +impl Assembler { + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } /// Get the list of registers from which we can allocate on this platform pub fn get_alloc_regs() -> Vec<Reg> { @@ -109,17 +127,23 @@ impl Assembler vec![RAX_REG, RCX_REG, RDX_REG, RSI_REG, RDI_REG, R8_REG, R9_REG, R10_REG, R11_REG] } + /// How many bytes a call and a bare bones [Self::frame_setup] would change native SP + pub fn frame_size() -> i32 { + 0x10 + } + // These are the callee-saved registers in the x86-64 SysV ABI // RBX, RSP, RBP, and R12–R15 /// Split IR instructions for the x86 platform fn x86_split(mut self) -> Assembler { + let mut asm_local = Assembler::new_with_asm(&self); + let asm = &mut asm_local; let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); - let mut iterator = self.insns.into_iter().enumerate().peekable(); - let mut asm = Assembler::new_with_label_names(take(&mut self.label_names), live_ranges.len()); + let mut iterator = self.instruction_iterator(); - while let Some((index, mut insn)) = iterator.next() { + while let Some((index, mut insn)) = iterator.next(asm) { let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. }); let mut opnd_iter = insn.opnd_iter_mut(); @@ -139,6 +163,15 @@ impl Assembler }; } + // When we split an operand, we can create a new VReg not in `live_ranges`. + // So when we see a VReg with out-of-range index, it's created from splitting + // from the loop above and we know it doesn't outlive the current instruction. + let vreg_outlives_insn = |vreg_idx| { + live_ranges + .get(vreg_idx) + .is_some_and(|live_range: &LiveRange| live_range.end() > index) + }; + // We are replacing instructions here so we know they are already // being used. It is okay not to use their output here. #[allow(unused_must_use)] @@ -155,13 +188,13 @@ impl Assembler if out == src && left == dest && live_ranges[out.vreg_idx()].end() == index + 1 && uimm_num_bits(*value) <= 32 => { *out = *dest; asm.push_insn(insn); - iterator.next(); // Pop merged Insn::Mov + iterator.next(asm); // Pop merged Insn::Mov } (Opnd::Reg(_), Opnd::Reg(_), Some(Insn::Mov { dest, src })) if out == src && live_ranges[out.vreg_idx()].end() == index + 1 && *dest == *left => { *out = *dest; asm.push_insn(insn); - iterator.next(); // Pop merged Insn::Mov + iterator.next(asm); // Pop merged Insn::Mov } _ => { match (*left, *right) { @@ -174,14 +207,20 @@ impl Assembler }, // Instruction output whose live range spans beyond this instruction (Opnd::VReg { idx, .. }, _) => { - if live_ranges[idx].end() > index { + if vreg_outlives_insn(idx) { *left = asm.load(*left); } }, // We have to load memory operands to avoid corrupting them - (Opnd::Mem(_) | Opnd::Reg(_), _) => { + (Opnd::Mem(_), _) => { *left = asm.load(*left); }, + // We have to load register operands to avoid corrupting them + (Opnd::Reg(_), _) => { + if *left != *out { + *left = asm.load(*left); + } + }, // The first operand can't be an immediate value (Opnd::UImm(_), _) => { *left = asm.load(*left); @@ -239,12 +278,12 @@ impl Assembler match opnd { // Instruction output whose live range spans beyond this instruction Opnd::VReg { idx, .. } => { - if live_ranges[*idx].end() > index { + if vreg_outlives_insn(*idx) { *opnd = asm.load(*opnd); } }, - // We have to load memory operands to avoid corrupting them - Opnd::Mem(_) | Opnd::Reg(_) => { + // We have to load non-reg operands to avoid corrupting them + Opnd::Mem(_) | Opnd::Reg(_) | Opnd::UImm(_) | Opnd::Imm(_) => { *opnd = asm.load(*opnd); }, _ => {} @@ -263,7 +302,7 @@ impl Assembler // If we have an instruction output whose live range // spans beyond this instruction, we have to load it. Opnd::VReg { idx, .. } => { - if live_ranges[idx].end() > index { + if vreg_outlives_insn(idx) { *truthy = asm.load(*truthy); } }, @@ -286,33 +325,11 @@ impl Assembler asm.push_insn(insn); }, - Insn::Mov { dest, src } | Insn::Store { dest, src } => { - match (&dest, &src) { - (Opnd::Mem(_), Opnd::Mem(_)) => { - // We load opnd1 because for mov, opnd0 is the output - let opnd1 = asm.load(*src); - asm.mov(*dest, opnd1); - }, - (Opnd::Mem(_), Opnd::UImm(value)) => { - // 32-bit values will be sign-extended - if imm_num_bits(*value as i64) > 32 { - let opnd1 = asm.load(*src); - asm.mov(*dest, opnd1); - } else { - asm.mov(*dest, *src); - } - }, - (Opnd::Mem(_), Opnd::Imm(value)) => { - if imm_num_bits(*value) > 32 { - let opnd1 = asm.load(*src); - asm.mov(*dest, opnd1); - } else { - asm.mov(*dest, *src); - } - }, - _ => { - asm.mov(*dest, *src); - } + Insn::Mov { dest, src } => { + if let Opnd::Mem(_) = dest { + asm.store(*dest, *src); + } else { + asm.mov(*dest, *src); } }, Insn::Not { opnd, .. } => { @@ -320,7 +337,7 @@ impl Assembler // If we have an instruction output whose live range // spans beyond this instruction, we have to load it. Opnd::VReg { idx, .. } => { - if live_ranges[idx].end() > index { + if vreg_outlives_insn(idx) { *opnd = asm.load(*opnd); } }, @@ -339,9 +356,9 @@ impl Assembler // Load each operand into the corresponding argument register. if !opnds.is_empty() { - let mut args: Vec<(Reg, Opnd)> = vec![]; - for (idx, opnd) in opnds.into_iter().enumerate() { - args.push((C_ARG_OPNDS[idx].unwrap_reg(), *opnd)); + let mut args: Vec<(Opnd, Opnd)> = vec![]; + for (idx, opnd) in opnds.iter_mut().enumerate() { + args.push((C_ARG_OPNDS[idx], *opnd)); } asm.parallel_mov(args); } @@ -357,7 +374,7 @@ impl Assembler (Insn::Lea { opnd, out }, Some(Insn::Mov { dest: Opnd::Reg(reg), src })) if matches!(out, Opnd::VReg { .. }) && out == src && live_ranges[out.vreg_idx()].end() == index + 1 => { asm.push_insn(Insn::Lea { opnd: *opnd, out: Opnd::Reg(*reg) }); - iterator.next(); // Pop merged Insn::Mov + iterator.next(asm); // Pop merged Insn::Mov } _ => asm.push_insn(insn), } @@ -368,40 +385,275 @@ impl Assembler } } - asm + asm_local } - /// Emit platform-specific machine code - pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Option<Vec<u32>> - { + /// Split instructions using scratch registers. To maximize the use of the register pool + /// for VRegs, most splits should happen in [`Self::x86_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so + /// this splits them and uses scratch registers for it. + pub fn x86_scratch_split(mut self) -> Assembler { /// For some instructions, we want to be able to lower a 64-bit operand /// without requiring more registers to be available in the register - /// allocator. So we just use the SCRATCH0 register temporarily to hold + /// allocator. So we just use the SCRATCH0_OPND register temporarily to hold /// the value before we immediately use it. - fn emit_64bit_immediate(cb: &mut CodeBlock, opnd: &Opnd) -> X86Opnd { + fn split_64bit_immediate(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { match opnd { Opnd::Imm(value) => { // 32-bit values will be sign-extended - if imm_num_bits(*value) > 32 { - mov(cb, Assembler::SCRATCH0, opnd.into()); - Assembler::SCRATCH0 + if imm_num_bits(value) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd } else { - opnd.into() + opnd } }, Opnd::UImm(value) => { // 32-bit values will be sign-extended - if imm_num_bits(*value as i64) > 32 { - mov(cb, Assembler::SCRATCH0, opnd.into()); - Assembler::SCRATCH0 + if imm_num_bits(value as i64) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd } else { - opnd.into() + Opnd::Imm(value as i64) } }, - _ => opnd.into() + _ => opnd + } + } + + /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg using a scratch regsiter. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + if let Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp, num_bits }) = opnd { + let base = Opnd::Mem(stack_state.stack_membase_to_mem(stack_membase)); + asm.load_into(scratch_opnd, base); + Opnd::Mem(Mem { base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), disp, num_bits }) + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + /// If both opnd and other are Opnd::Mem, split opnd with scratch_opnd. + fn split_if_both_memory(asm: &mut Assembler, opnd: Opnd, other: Opnd, scratch_opnd: Opnd) -> Opnd { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (opnd, other) { + asm.load_into(scratch_opnd.with_num_bits(opnd.rm_num_bits()), opnd); + scratch_opnd.with_num_bits(opnd.rm_num_bits()) + } else { + opnd + } + } + + /// Move src to dst, splitting it with scratch_opnd if it's a Mem-to-Mem move. Skip it if dst == src. + fn asm_mov(asm: &mut Assembler, dst: Opnd, src: Opnd, scratch_opnd: Opnd) { + if dst != src { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (dst, src) { + asm.mov(scratch_opnd, src); + asm.mov(dst, scratch_opnd); + } else { + asm.mov(dst, src); + } + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new_with_asm(&self); + let asm = &mut asm_local; + asm.accept_scratch_reg = true; + let mut iterator = self.instruction_iterator(); + + while let Some((_, mut insn)) = iterator.next(asm) { + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + *left = split_if_both_memory(asm, *left, *right, SCRATCH0_OPND); + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + + let (out, left) = (*out, *left); + asm.push_insn(insn); + asm_mov(asm, out, left, SCRATCH0_OPND); + } + Insn::Mul { left, right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + *left = split_if_both_memory(asm, *left, *right, SCRATCH0_OPND); + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + + // imul doesn't have (Mem, Reg) encoding. Swap left and right in that case. + if let (Opnd::Mem(_), Opnd::Reg(_)) = (&left, &right) { + mem::swap(left, right); + } + + let (out, left) = (*out, *left); + asm.push_insn(insn); + asm_mov(asm, out, left, SCRATCH0_OPND); + } + &mut Insn::Not { opnd, out } | + &mut Insn::LShift { opnd, out, .. } | + &mut Insn::RShift { opnd, out, .. } | + &mut Insn::URShift { opnd, out, .. } => { + asm.push_insn(insn); + asm_mov(asm, out, opnd, SCRATCH0_OPND); + } + Insn::Test { left, right } | + Insn::Cmp { left, right } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + + let num_bits = match right { + Opnd::Imm(value) => Some(imm_num_bits(*value)), + Opnd::UImm(value) => Some(uimm_num_bits(*value)), + _ => None + }; + + // If the immediate is less than 64 bits (like 32, 16, 8), and the operand + // sizes match, then we can represent it as an immediate in the instruction + // without moving it to a register first. + // IOW, 64 bit immediates must always be moved to a register + // before comparisons, where other sizes may be encoded + // directly in the instruction. + let use_imm = num_bits.is_some() && left.num_bits() == num_bits && num_bits.unwrap() < 64; + if !use_imm { + *right = split_64bit_immediate(asm, *right, SCRATCH0_OPND); + } + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Lea { opnd, out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::LeaJumpTarget { target, out } => { + if let Target::Label(_) = target { + asm.push_insn(Insn::LeaJumpTarget { out: SCRATCH0_OPND, target: target.clone() }); + asm.mov(*out, SCRATCH0_OPND); + } + } + Insn::Load { out, opnd } | + Insn::LoadInto { dest: out, opnd } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND.with_num_bits(mem_out.rm_num_bits())); + } + } + // Convert Opnd::const_ptr into Opnd::Mem. This split is done here to give + // a register for compile_exits. + &mut Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.incr_counter(Opnd::mem(64, SCRATCH0_OPND, 0), value); + } + &mut Insn::Mov { dest, src } => { + asm_mov(asm, dest, src, SCRATCH0_OPND); + } + // Resolve ParallelMov that couldn't be handled without a scratch register. + Insn::ParallelMov { moves } => { + for (dst, src) in Self::resolve_parallel_moves(&moves, Some(SCRATCH0_OPND)).unwrap() { + asm_mov(asm, dst, src, SCRATCH0_OPND); + } + } + // Handle various operand combinations for spills on compile_exits. + &mut Insn::Store { dest, src } => { + let num_bits = dest.rm_num_bits(); + let dest = split_stack_membase(asm, dest, SCRATCH1_OPND, &stack_state); + + let src = match src { + Opnd::Reg(_) => src, + Opnd::Mem(_) => { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + Opnd::Imm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else if uimm_num_bits(imm as u64) <= num_bits { + // If the bit string is short enough for the destination, use the unsigned representation. + // Note that 64-bit and negative values are ruled out. + Opnd::UImm(imm as u64) + } else { + src + } + } + Opnd::UImm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm as i64) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else { + src.into() + } + } + Opnd::Value(_) => { + asm.load_into(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + src @ (Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during x86_scratch_split: {src:?}"), + }; + asm.store(dest, src); + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } } } + asm_local + } + + /// Emit platform-specific machine code + pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Option<Vec<CodePtr>> { fn emit_csel( cb: &mut CodeBlock, truthy: Opnd, @@ -429,22 +681,31 @@ impl Assembler } } - //dbg!(&self.insns); + fn emit_load_gc_value(cb: &mut CodeBlock, gc_offsets: &mut Vec<CodePtr>, dest_reg: X86Opnd, value: VALUE) { + // Using movabs because mov might write value in 32 bits + movabs(cb, dest_reg, value.0 as _); + // The pointer immediate is encoded as the last part of the mov written out + let ptr_offset = cb.get_write_ptr().sub_bytes(SIZEOF_VALUE); + gc_offsets.push(ptr_offset); + } // List of GC offsets - let mut gc_offsets: Vec<u32> = Vec::new(); + let mut gc_offsets: Vec<CodePtr> = Vec::new(); // Buffered list of PosMarker callbacks to fire if codegen is successful let mut pos_markers: Vec<(usize, CodePtr)> = vec![]; + // The write_pos for the last Insn::PatchPoint, if any + let mut last_patch_pos: Option<usize> = None; + + // Install a panic hook to dump Assembler with insn_idx on dev builds + let (_hook, mut hook_insn_idx) = AssemblerPanicHook::new(self, 0); + // For each instruction - //let start_write_pos = cb.get_write_pos(); let mut insn_idx: usize = 0; while let Some(insn) = self.insns.get(insn_idx) { - //let src_ptr = cb.get_write_ptr(); - let had_dropped_bytes = cb.has_dropped_bytes(); - //let old_label_state = cb.get_label_state(); - let mut insn_gc_offsets: Vec<u32> = Vec::new(); + // Update insn_idx that is shown on panic + hook_insn_idx.as_mut().map(|idx| idx.lock().map(|mut idx| *idx = insn_idx).unwrap()); match insn { Insn::Comment(text) => { @@ -471,50 +732,57 @@ impl Assembler cb.write_byte(0); }, - // Set up RBP to work with frame pointer unwinding + // Set up RBP as frame pointer work with unwinding // (e.g. with Linux `perf record --call-graph fp`) - Insn::FrameSetup => { - if false { // We don't support --zjit-perf yet - push(cb, RBP); - mov(cb, RBP, RSP); - push(cb, RBP); + // and to allow push and pops in the function. + &Insn::FrameSetup { preserved, mut slot_count } => { + // Bump slot_count for alignment if necessary + const { assert!(SIZEOF_VALUE == 8, "alignment logic relies on SIZEOF_VALUE == 8"); } + let total_slots = 2 /* rbp and return address*/ + slot_count + preserved.len(); + if total_slots % 2 == 1 { + slot_count += 1; } - }, - Insn::FrameTeardown => { - if false { // We don't support --zjit-perf yet - pop(cb, RBP); - pop(cb, RBP); + push(cb, RBP); + mov(cb, RBP, RSP); + for reg in preserved { + push(cb, reg.into()); } - }, + if slot_count > 0 { + sub(cb, RSP, uimm_opnd((slot_count * SIZEOF_VALUE) as u64)); + } + } + &Insn::FrameTeardown { preserved } => { + let mut preserved_offset = -8; + for reg in preserved { + mov(cb, reg.into(), mem_opnd(64, RBP, preserved_offset)); + preserved_offset -= 8; + } + mov(cb, RSP, RBP); + pop(cb, RBP); + } Insn::Add { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - add(cb, left.into(), opnd1); + add(cb, left.into(), right.into()); }, Insn::Sub { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - sub(cb, left.into(), opnd1); + sub(cb, left.into(), right.into()); }, Insn::Mul { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - imul(cb, left.into(), opnd1); + imul(cb, left.into(), right.into()); }, Insn::And { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - and(cb, left.into(), opnd1); + and(cb, left.into(), right.into()); }, Insn::Or { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - or(cb, left.into(), opnd1); + or(cb, left.into(), right.into()); }, Insn::Xor { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - xor(cb, left.into(), opnd1); + xor(cb, left.into(), right.into()); }, Insn::Not { opnd, .. } => { @@ -533,20 +801,12 @@ impl Assembler shr(cb, opnd.into(), shift.into()) }, - Insn::Store { dest, src } => { - mov(cb, dest.into(), src.into()); - }, - // This assumes only load instructions can contain references to GC'd Value operands Insn::Load { opnd, out } | Insn::LoadInto { dest: out, opnd } => { match opnd { Opnd::Value(val) if val.heap_object_p() => { - // Using movabs because mov might write value in 32 bits - movabs(cb, out.into(), val.0 as _); - // The pointer immediate is encoded as the last part of the mov written out - let ptr_offset: u32 = (cb.get_write_pos() as u32) - (SIZEOF_VALUE as u32); - insn_gc_offsets.push(ptr_offset); + emit_load_gc_value(cb, &mut gc_offsets, out.into(), *val); } _ => mov(cb, out.into(), opnd.into()) } @@ -558,6 +818,7 @@ impl Assembler Insn::ParallelMov { .. } => unreachable!("{insn:?} should have been lowered at alloc_regs()"), + Insn::Store { dest, src } | Insn::Mov { dest, src } => { mov(cb, dest.into(), src.into()); }, @@ -570,13 +831,12 @@ impl Assembler // Load address of jump target Insn::LeaJumpTarget { target, out } => { if let Target::Label(label) = target { + let out = *out; // Set output to the raw address of the label - cb.label_ref(*label, 7, |cb, src_addr, dst_addr| { + cb.label_ref(*label, 7, move |cb, src_addr, dst_addr| { let disp = dst_addr - src_addr; - lea(cb, Self::SCRATCH0, mem_opnd(8, RIP, disp.try_into().unwrap())); + lea(cb, out.into(), mem_opnd(8, RIP, disp.try_into().unwrap())); }); - - mov(cb, out.into(), Self::SCRATCH0); } else { // Set output to the jump target's raw address let target_code = target.unwrap_code_ptr(); @@ -618,7 +878,15 @@ impl Assembler // C function call Insn::CCall { fptr, .. } => { - call_ptr(cb, RAX, *fptr); + match fptr { + Opnd::UImm(fptr) => { + call_ptr(cb, RAX, *fptr as *const u8); + } + Opnd::Reg(_) => { + call(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") + } }, Insn::CRet(opnd) => { @@ -632,30 +900,12 @@ impl Assembler // Compare Insn::Cmp { left, right } => { - let num_bits = match right { - Opnd::Imm(value) => Some(imm_num_bits(*value)), - Opnd::UImm(value) => Some(uimm_num_bits(*value)), - _ => None - }; - - // If the immediate is less than 64 bits (like 32, 16, 8), and the operand - // sizes match, then we can represent it as an immediate in the instruction - // without moving it to a register first. - // IOW, 64 bit immediates must always be moved to a register - // before comparisons, where other sizes may be encoded - // directly in the instruction. - if num_bits.is_some() && left.num_bits() == num_bits && num_bits.unwrap() < 64 { - cmp(cb, left.into(), right.into()); - } else { - let emitted = emit_64bit_immediate(cb, right); - cmp(cb, left.into(), emitted); - } + cmp(cb, left.into(), right.into()); } // Test and set flags Insn::Test { left, right } => { - let emitted = emit_64bit_immediate(cb, right); - test(cb, left.into(), emitted); + test(cb, left.into(), right.into()); } Insn::JmpOpnd(opnd) => { @@ -665,95 +915,107 @@ impl Assembler // Conditional jump to a label Insn::Jmp(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jmp_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr), Target::Label(label) => jmp_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } Insn::Je(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => je_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr), Target::Label(label) => je_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } Insn::Jne(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jne_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr), Target::Label(label) => jne_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } Insn::Jl(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jl_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr), Target::Label(label) => jl_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, Insn::Jg(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jg_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jg_ptr(cb, code_ptr), Target::Label(label) => jg_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, Insn::Jge(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jge_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jge_ptr(cb, code_ptr), Target::Label(label) => jge_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, Insn::Jbe(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jbe_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr), Target::Label(label) => jbe_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, Insn::Jb(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jb_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jb_ptr(cb, code_ptr), Target::Label(label) => jb_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, Insn::Jz(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jz_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr), Target::Label(label) => jz_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } Insn::Jnz(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jnz_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr), Target::Label(label) => jnz_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } Insn::Jo(target) | Insn::JoMul(target) => { match *target { - Target::CodePtr(code_ptr) | Target::SideExitPtr(code_ptr) => jo_ptr(cb, code_ptr), + Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr), Target::Label(label) => jo_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } Insn::Joz(..) | Insn::Jonz(..) => unreachable!("Joz/Jonz should be unused for now"), + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in x86_scratch_split"), + Insn::PadPatchPoint => { + // If patch points are too close to each other or the end of the block, fill nop instructions + if let Some(last_patch_pos) = last_patch_pos { + let code_size = cb.get_write_pos().saturating_sub(last_patch_pos); + if code_size < cb.jmp_ptr_bytes() { + nop(cb, (cb.jmp_ptr_bytes() - code_size) as u32); + } + } + last_patch_pos = Some(cb.get_write_pos()); + }, + // Atomically increment a counter at a given memory location Insn::IncrCounter { mem, value } => { assert!(matches!(mem, Opnd::Mem(_))); @@ -789,48 +1051,48 @@ impl Assembler emit_csel(cb, *truthy, *falsy, *out, cmovge, cmovl); } Insn::LiveReg { .. } => (), // just a reg alloc signal, no code - Insn::PadInvalPatch => { - unimplemented!("we don't need padding yet"); - /* - let code_size = cb.get_write_pos().saturating_sub(std::cmp::max(start_write_pos, cb.page_start_pos())); - if code_size < cb.jmp_ptr_bytes() { - nop(cb, (cb.jmp_ptr_bytes() - code_size) as u32); - } - */ - } }; - // On failure, jump to the next page and retry the current insn - if !had_dropped_bytes && cb.has_dropped_bytes() { - // Reset cb states before retrying the current Insn - //cb.set_label_state(old_label_state); - } else { - insn_idx += 1; - gc_offsets.append(&mut insn_gc_offsets); - } + insn_idx += 1; } // Error if we couldn't write out everything if cb.has_dropped_bytes() { - return None + None } else { // No bytes dropped, so the pos markers point to valid code for (insn_idx, pos) in pos_markers { if let Insn::PosMarker(callback) = self.insns.get(insn_idx).unwrap() { - callback(pos, &cb); + callback(pos, cb); } else { panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); } } - return Some(gc_offsets) + Some(gc_offsets) } } /// Optimize and compile the stored instructions - pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Option<(CodePtr, Vec<u32>)> { + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_regs = !self.accept_scratch_reg; + asm_dump!(self, init); + let asm = self.x86_split(); - let mut asm = asm.alloc_regs(regs); + asm_dump!(asm, split); + + let mut asm = asm.alloc_regs(regs)?; + asm_dump!(asm, alloc_regs); + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + asm.compile_exits(); + asm_dump!(asm, compile_exits); + + if use_scratch_regs { + asm = asm.x86_scratch_split(); + asm_dump!(asm, scratch_split); + } // Create label instances in the code block for (idx, name) in asm.label_names.iter().enumerate() { @@ -843,67 +1105,129 @@ impl Assembler if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { cb.link_labels(); - - Some((start_ptr, gc_offsets)) + Ok((start_ptr, gc_offsets)) } else { cb.clear_labels(); - - None + Err(CompileError::OutOfMemory) } } } -/* #[cfg(test)] mod tests { - use crate::disasm::assert_disasm; - #[cfg(feature = "disasm")] - use crate::disasm::{unindent, disasm_addr_range}; - + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; + use crate::options::rb_zjit_prepare_options; use super::*; + const BOLD_BEGIN: &str = "\x1b[1m"; + const BOLD_END: &str = "\x1b[22m"; + fn setup_asm() -> (Assembler, CodeBlock) { - (Assembler::new(0), CodeBlock::new_dummy(1024)) + rb_zjit_prepare_options(); // for get_option! on asm.compile + (Assembler::new(), CodeBlock::new_dummy()) } #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.parallel_mov(vec![(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)), (C_ARG_OPNDS[1], Opnd::mem(64, SP, -8))]); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.je(label); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @r" + bb0: + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, r13, rbx, r12 + v0 = Add r13, 0x40 + Store [rbx + 0x10], v0 + Joz Exit(Interrupt), v0 + ParallelMov rdi <- eax, rsi <- [rbx - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[r12 + 0x10], VReg32(v1) + Je bb0 + CRet v0 + FrameTeardown r13, rbx, r12 + "); + } + + #[test] + #[ignore] fn test_emit_add_lt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881c0ff000000"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: add rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c0ff000000"); } #[test] + #[ignore] fn test_emit_add_gt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c01d8"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: add rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c01d8"); } #[test] + #[ignore] fn test_emit_and_lt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881e0ff000000"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: and rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e0ff000000"); } #[test] + #[ignore] fn test_emit_and_gt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c21d8"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: and rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c21d8"); } #[test] @@ -913,7 +1237,8 @@ mod tests { asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "4881f8ff000000"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"4881f8ff000000"); } #[test] @@ -923,7 +1248,22 @@ mod tests { asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "49bbffffffffffff00004c39d8"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); + } + + #[test] + fn test_emit_cmp_64_bits() { + let (mut asm, mut cb) = setup_asm(); + + asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF_FFFF)); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, -1"); + assert_snapshot!(cb.hexdump(), @"4883f8ff"); } #[test] @@ -935,7 +1275,8 @@ mod tests { asm.cmp(shape_opnd, Opnd::UImm(0xF000)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "6681780600f0"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp word ptr [rax + 6], 0xf000"); + assert_snapshot!(cb.hexdump(), @"6681780600f0"); } #[test] @@ -947,47 +1288,70 @@ mod tests { asm.cmp(shape_opnd, Opnd::UImm(0xF000_0000)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "817804000000f0"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp dword ptr [rax + 4], 0xf0000000"); + assert_snapshot!(cb.hexdump(), @"817804000000f0"); } #[test] + #[ignore] fn test_emit_or_lt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881c8ff000000"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: or rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c8ff000000"); } #[test] + #[ignore] fn test_emit_or_gt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c09d8"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: or rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c09d8"); } #[test] + #[ignore] fn test_emit_sub_lt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881e8ff000000"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: sub rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e8ff000000"); } #[test] + #[ignore] fn test_emit_sub_gt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c29d8"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: sub rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c29d8"); } #[test] @@ -997,7 +1361,8 @@ mod tests { asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "f6c0ff"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: test rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"48f7c0ff000000"); } #[test] @@ -1007,27 +1372,42 @@ mod tests { asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "49bbffffffffffff00004c85d8"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: movabs r11, 0xffffffffffff + 0xa: test rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c85d8"); } #[test] + #[ignore] fn test_emit_xor_lt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881f0ff000000"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: xor rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881f0ff000000"); } #[test] + #[ignore] fn test_emit_xor_gt_32_bits() { let (mut asm, mut cb) = setup_asm(); let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c31d8"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: xor rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c31d8"); } #[test] @@ -1038,12 +1418,12 @@ mod tests { asm.mov(SP, sp); // should be merged to lea asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "488d5b08", {" - 0x0: lea rbx, [rbx + 8] - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lea rbx, [rbx + 8]"); + assert_snapshot!(cb.hexdump(), @"488d5b08"); } #[test] + #[ignore] fn test_merge_lea_mem() { let (mut asm, mut cb) = setup_asm(); @@ -1051,13 +1431,15 @@ mod tests { asm.mov(Opnd::mem(64, SP, 0), sp); // should NOT be merged to lea asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "488d4308488903", {" - 0x0: lea rax, [rbx + 8] - 0x4: mov qword ptr [rbx], rax - "}); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); } #[test] + #[ignore] fn test_replace_cmp_0() { let (mut asm, mut cb) = setup_asm(); @@ -1067,7 +1449,15 @@ mod tests { asm.mov(Opnd::Reg(RAX_REG), result); asm.compile_with_num_regs(&mut cb, 2); - assert_eq!(format!("{:x}", cb), "488b43084885c0b814000000b900000000480f45c14889c0"); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov rax, qword ptr [rbx + 8] + 0x4: test rax, rax + 0x7: mov eax, 0x14 + 0xc: mov ecx, 0 + 0x11: cmovne rax, rcx + 0x15: mov rax, rax + "); + assert_snapshot!(cb.hexdump(), @"488b43084885c0b814000000b900000000480f45c14889c0"); } #[test] @@ -1078,7 +1468,19 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983c540"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); + } + + #[test] + fn test_add_into() { + let (mut asm, mut cb) = setup_asm(); + + asm.add_into(CFP, Opnd::UImm(0x40)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); } #[test] @@ -1089,7 +1491,19 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983ed40"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); + } + + #[test] + fn test_sub_into() { + let (mut asm, mut cb) = setup_asm(); + + asm.sub_into(CFP, Opnd::UImm(0x40)); + asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); } #[test] @@ -1100,7 +1514,8 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983e540"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983e540"); } #[test] @@ -1111,7 +1526,8 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983cd40"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983cd40"); } #[test] @@ -1122,27 +1538,31 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983f540"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983f540"); } #[test] - fn test_reorder_c_args_no_cycle() { + fn test_ccall_resolve_parallel_moves_no_cycle() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); asm.ccall(0 as _, vec![ C_ARG_OPNDS[0], // mov rdi, rdi (optimized away) C_ARG_OPNDS[1], // mov rsi, rsi (optimized away) ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "b800000000ffd0", {" - 0x0: mov eax, 0 - 0x5: call rax - "}); + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: mov eax, 0 + 0x5: call rax + "); + assert_snapshot!(cb.hexdump(), @"b800000000ffd0"); } #[test] - fn test_reorder_c_args_single_cycle() { + fn test_ccall_resolve_parallel_moves_single_cycle() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); // rdi and rsi form a cycle @@ -1151,19 +1571,21 @@ mod tests { C_ARG_OPNDS[0], // mov rsi, rdi C_ARG_OPNDS[2], // mov rdx, rdx (optimized away) ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "4989f34889fe4c89dfb800000000ffd0", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov r11, rsi 0x3: mov rsi, rdi 0x6: mov rdi, r11 0x9: mov eax, 0 0xe: call rax - "}); + "); + assert_snapshot!(cb.hexdump(), @"4989f34889fe4c89dfb800000000ffd0"); } #[test] - fn test_reorder_c_args_two_cycles() { + fn test_ccall_resolve_parallel_moves_two_cycles() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); // rdi and rsi form a cycle, and rdx and rcx form another cycle @@ -1173,9 +1595,9 @@ mod tests { C_ARG_OPNDS[3], // mov rdx, rcx C_ARG_OPNDS[2], // mov rcx, rdx ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "4989f34889fe4c89df4989cb4889d14c89dab800000000ffd0", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov r11, rsi 0x3: mov rsi, rdi 0x6: mov rdi, r11 @@ -1184,11 +1606,13 @@ mod tests { 0xf: mov rdx, r11 0x12: mov eax, 0 0x17: call rax - "}); + "); + assert_snapshot!(cb.hexdump(), @"4989f34889fe4c89df4989cb4889d14c89dab800000000ffd0"); } #[test] - fn test_reorder_c_args_large_cycle() { + fn test_ccall_resolve_parallel_moves_large_cycle() { + crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); // rdi, rsi, and rdx form a cycle @@ -1197,20 +1621,22 @@ mod tests { C_ARG_OPNDS[2], // mov rsi, rdx C_ARG_OPNDS[0], // mov rdx, rdi ]); - asm.compile_with_num_regs(&mut cb, 0); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "4989f34889d64889fa4c89dfb800000000ffd0", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov r11, rsi 0x3: mov rsi, rdx 0x6: mov rdx, rdi 0x9: mov rdi, r11 0xc: mov eax, 0 0x11: call rax - "}); + "); + assert_snapshot!(cb.hexdump(), @"4989f34889d64889fa4c89dfb800000000ffd0"); } #[test] - fn test_reorder_c_args_with_insn_out() { + #[ignore] + fn test_ccall_resolve_parallel_moves_with_insn_out() { let (mut asm, mut cb) = setup_asm(); let rax = asm.load(Opnd::UImm(1)); @@ -1225,7 +1651,7 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, 3); - assert_disasm!(cb, "b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov eax, 1 0x5: mov ecx, 2 0xa: mov edx, 3 @@ -1236,7 +1662,8 @@ mod tests { 0x1b: mov rdx, r11 0x1e: mov eax, 0 0x23: call rax - "}); + "); + assert_snapshot!(cb.hexdump(), @"b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0"); } #[test] @@ -1253,15 +1680,17 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "48837b1001b804000000480f4f03488903", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp qword ptr [rbx + 0x10], 1 - 0x5: mov eax, 4 - 0xa: cmovg rax, qword ptr [rbx] - 0xe: mov qword ptr [rbx], rax - "}); + 0x5: mov edi, 4 + 0xa: cmovg rdi, qword ptr [rbx] + 0xe: mov qword ptr [rbx], rdi + "); + assert_snapshot!(cb.hexdump(), @"48837b1001bf04000000480f4f3b48893b"); } #[test] + #[ignore] fn test_csel_split() { let (mut asm, mut cb) = setup_asm(); @@ -1271,13 +1700,126 @@ mod tests { asm.compile_with_num_regs(&mut cb, 3); - assert_disasm!(cb, "48b830198dc8227f0000b904000000480f44c1488903", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: movabs rax, 0x7f22c88d1930 0xa: mov ecx, 4 0xf: cmove rax, rcx 0x13: mov qword ptr [rbx], rax - "}); + "); + assert_snapshot!(cb.hexdump(), @"48b830198dc8227f0000b904000000480f44c1488903"); } -} -*/ + #[test] + fn test_mov_m32_imm32() { + let (mut asm, mut cb) = setup_asm(); + + let shape_opnd = Opnd::mem(32, C_RET_OPND, 0); + asm.mov(shape_opnd, Opnd::UImm(0x8000_0001)); + asm.mov(shape_opnd, Opnd::Imm(0x8000_0001)); + + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov dword ptr [rax], 0x80000001 + 0x6: mov dword ptr [rax], 0x80000001 + "); + assert_snapshot!(cb.hexdump(), @"c70001000080c70001000080"); + } + + #[test] + fn frame_setup_teardown_preserved_regs() { + let (mut asm, mut cb) = setup_asm(); + asm.frame_setup(JIT_PRESERVED_REGS); + asm.frame_teardown(JIT_PRESERVED_REGS); + asm.cret(C_RET_OPND); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: push r13 + 0x6: push rbx + 0x7: push r12 + 0x9: sub rsp, 8 + 0xd: mov r13, qword ptr [rbp - 8] + 0x11: mov rbx, qword ptr [rbp - 0x10] + 0x15: mov r12, qword ptr [rbp - 0x18] + 0x19: mov rsp, rbp + 0x1c: pop rbp + 0x1d: ret + "); + assert_snapshot!(cb.hexdump(), @"554889e541555341544883ec084c8b6df8488b5df04c8b65e84889ec5dc3"); + } + + #[test] + fn frame_setup_teardown_stack_base_idx() { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 5; + asm.frame_setup(&[]); + asm.frame_teardown(&[]); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @r" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: sub rsp, 0x30 + 0x8: mov rsp, rbp + 0xb: pop rbp + "); + assert_snapshot!(cb.hexdump(), @"554889e54883ec304889ec5d"); + } + + #[test] + fn test_store_value_without_split() { + let (mut asm, mut cb) = setup_asm(); + + let imitation_heap_value = VALUE(0x1000); + assert!(imitation_heap_value.heap_object_p()); + asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into()); + + asm = asm.x86_scratch_split(); + let gc_offsets = asm.x86_emit(&mut cb).unwrap(); + assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0x1000 + 0xa: mov qword ptr [rbx], r11 + "); + assert_snapshot!(cb.hexdump(), @"49bb00100000000000004c891b"); + } + + #[test] + fn test_csel_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let left = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let right = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 1, num_bits: 64 }, disp: 2, num_bits: 64 }); + let _ = asm.csel_e(left, right); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r10, qword ptr [rbp - 8] + 0x4: mov r11, qword ptr [rbp - 0x10] + 0x8: mov r11, qword ptr [r11 + 2] + 0xc: cmove r11, qword ptr [r10] + 0x10: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8b55f84c8b5df04d8b5b024d0f441a4c895df8"); + } + + #[test] + fn test_lea_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let _ = asm.lea(opnd); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, qword ptr [rbp - 8] + 0x4: lea r11, [r11] + 0x7: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8b5df84d8d1b4c895df8"); + } +} diff --git a/zjit/src/bitset.rs b/zjit/src/bitset.rs new file mode 100644 index 0000000000..b5b69abdee --- /dev/null +++ b/zjit/src/bitset.rs @@ -0,0 +1,126 @@ +//! Optimized bitset implementation. + +type Entry = u128; + +const ENTRY_NUM_BITS: usize = Entry::BITS as usize; + +// TODO(max): Make a `SmallBitSet` and `LargeBitSet` and switch between them if `num_bits` fits in +// `Entry`. +#[derive(Clone)] +pub struct BitSet<T: Into<usize> + Copy> { + entries: Vec<Entry>, + num_bits: usize, + phantom: std::marker::PhantomData<T>, +} + +impl<T: Into<usize> + Copy> BitSet<T> { + pub fn with_capacity(num_bits: usize) -> Self { + let num_entries = num_bits.div_ceil(ENTRY_NUM_BITS); + Self { entries: vec![0; num_entries], num_bits, phantom: Default::default() } + } + + /// Returns whether the value was newly inserted: true if the set did not originally contain + /// the bit, and false otherwise. + pub fn insert(&mut self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + let newly_inserted = (self.entries[entry_idx] & (1 << bit_idx)) == 0; + self.entries[entry_idx] |= 1 << bit_idx; + newly_inserted + } + + /// Set all bits to 1. + pub fn insert_all(&mut self) { + for i in 0..self.entries.len() { + self.entries[i] = !0; + } + } + + pub fn get(&self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + (self.entries[entry_idx] & (1 << bit_idx)) != 0 + } + + /// Modify `self` to only have bits set if they are also set in `other`. Returns true if `self` + /// was modified, and false otherwise. + /// `self` and `other` must have the same number of bits. + pub fn intersect_with(&mut self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + let mut changed = false; + for i in 0..self.entries.len() { + let before = self.entries[i]; + self.entries[i] &= other.entries[i]; + changed |= self.entries[i] != before; + } + changed + } +} + +#[cfg(test)] +mod tests { + use super::BitSet; + + #[test] + #[should_panic] + fn get_over_capacity_panics() { + let set = BitSet::with_capacity(0); + assert!(!set.get(0usize)); + } + + #[test] + fn with_capacity_defaults_to_zero() { + let set = BitSet::with_capacity(4); + assert!(!set.get(0usize)); + assert!(!set.get(1usize)); + assert!(!set.get(2usize)); + assert!(!set.get(3usize)); + } + + #[test] + fn insert_sets_bit() { + let mut set = BitSet::with_capacity(4); + assert!(set.insert(1usize)); + assert!(set.get(1usize)); + } + + #[test] + fn insert_with_set_bit_returns_false() { + let mut set = BitSet::with_capacity(4); + assert!(set.insert(1usize)); + assert!(!set.insert(1usize)); + } + + #[test] + fn insert_all_sets_all_bits() { + let mut set = BitSet::with_capacity(4); + set.insert_all(); + assert!(set.get(0usize)); + assert!(set.get(1usize)); + assert!(set.get(2usize)); + assert!(set.get(3usize)); + } + + #[test] + #[should_panic] + fn intersect_with_panics_with_different_num_bits() { + let mut left: BitSet<usize> = BitSet::with_capacity(3); + let right = BitSet::with_capacity(4); + left.intersect_with(&right); + } + #[test] + fn intersect_with_keeps_only_common_bits() { + let mut left = BitSet::with_capacity(3); + let mut right = BitSet::with_capacity(3); + left.insert(0usize); + left.insert(1usize); + right.insert(1usize); + right.insert(2usize); + left.intersect_with(&right); + assert!(!left.get(0usize)); + assert!(left.get(1usize)); + assert!(!left.get(2usize)); + } +} diff --git a/zjit/src/cast.rs b/zjit/src/cast.rs index bacc7245f3..52e2078cde 100644 --- a/zjit/src/cast.rs +++ b/zjit/src/cast.rs @@ -1,3 +1,7 @@ +//! Optimized [usize] casting trait. + +#![allow(clippy::wrong_self_convention)] + /// Trait for casting to [usize] that allows you to say `.as_usize()`. /// Implementation conditional on the cast preserving the numeric value on /// all inputs and being inexpensive. @@ -12,19 +16,19 @@ /// the method `into()` also causes a name conflict. pub(crate) trait IntoUsize { /// Convert to usize. Implementation conditional on width of [usize]. - fn as_usize(self) -> usize; + fn to_usize(self) -> usize; } #[cfg(target_pointer_width = "64")] impl IntoUsize for u64 { - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self as usize } } #[cfg(target_pointer_width = "64")] impl IntoUsize for u32 { - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self as usize } } @@ -32,7 +36,7 @@ impl IntoUsize for u32 { impl IntoUsize for u16 { /// Alias for `.into()`. For convenience so you could use the trait for /// all unsgined types. - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self.into() } } @@ -40,7 +44,7 @@ impl IntoUsize for u16 { impl IntoUsize for u8 { /// Alias for `.into()`. For convenience so you could use the trait for /// all unsgined types. - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self.into() } } diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 844ac5df42..7afcff5863 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -1,50 +1,77 @@ -use std::cell::Cell; -use std::rc::Rc; +//! This module is for native code generation. + +#![allow(clippy::let_and_return)] -use crate::backend::current::{Reg, ALLOC_REGS}; -use crate::profile::get_or_create_iseq_payload; +use std::cell::{Cell, RefCell}; +use std::rc::Rc; +use std::ffi::{c_int, c_long, c_void}; +use std::slice; + +use crate::backend::current::ALLOC_REGS; +use crate::invariants::{ + track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption, + track_single_ractor_assumption, track_stable_constant_names_assumption, track_no_singleton_class_assumption +}; +use crate::gc::append_gc_offsets; +use crate::payload::{get_or_create_iseq_payload, IseqCodePtrs, IseqVersion, IseqVersionRef, IseqStatus}; use crate::state::ZJITState; +use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; +use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::{compile_time_ns, exit_compile_error}}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; -use crate::invariants::{iseq_escapes_ep, track_no_ep_escape_assumption}; -use crate::backend::lir::{self, asm_comment, Assembler, Opnd, Target, CFP, C_ARG_OPNDS, C_RET_OPND, EC, SP}; -use crate::hir::{iseq_to_hir, Block, BlockId, BranchEdge, CallInfo}; -use crate::hir::{Const, FrameState, Function, Insn, InsnId}; -use crate::hir_type::{types::Fixnum, Type}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, NATIVE_STACK_PTR, Opnd, SP, SideExit, Target, asm_ccall, asm_comment}; +use crate::hir::{iseq_to_hir, BlockId, BranchEdge, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; +use crate::hir::{Const, FrameState, Function, Insn, InsnId, SendFallbackReason}; +use crate::hir_type::{types, Type}; use crate::options::get_option; +use crate::cast::IntoUsize; + +/// At the moment, we support recompiling each ISEQ only once. +pub const MAX_ISEQ_VERSIONS: usize = 2; + +/// Sentinel program counter stored in C frames when runtime checks are enabled. +const PC_POISON: Option<*const VALUE> = if cfg!(feature = "runtime_checks") { + Some(usize::MAX as *const VALUE) +} else { + None +}; /// Ephemeral code generation state struct JITState { /// Instruction sequence for the method being compiled iseq: IseqPtr, + /// ISEQ version that is being compiled, which will be used by PatchPoint + version: IseqVersionRef, + /// Low-level IR Operands indexed by High-level IR's Instruction ID opnds: Vec<Option<Opnd>>, /// Labels for each basic block indexed by the BlockId labels: Vec<Option<Target>>, - /// Branches to an ISEQ that need to be compiled later - branch_iseqs: Vec<(Rc<Branch>, IseqPtr)>, + /// JIT entry point for the `iseq` + jit_entries: Vec<Rc<RefCell<JITEntry>>>, + + /// ISEQ calls that need to be compiled later + iseq_calls: Vec<IseqCallRef>, } impl JITState { /// Create a new JITState instance - fn new(iseq: IseqPtr, num_insns: usize, num_blocks: usize) -> Self { + fn new(iseq: IseqPtr, version: IseqVersionRef, num_insns: usize, num_blocks: usize) -> Self { JITState { iseq, + version, opnds: vec![None; num_insns], labels: vec![None; num_blocks], - branch_iseqs: Vec::default(), + jit_entries: Vec::default(), + iseq_calls: Vec::default(), } } /// Retrieve the output of a given instruction that has been compiled - fn get_opnd(&self, insn_id: InsnId) -> Option<lir::Opnd> { - let opnd = self.opnds[insn_id.0]; - if opnd.is_none() { - debug!("Failed to get_opnd({insn_id})"); - } - opnd + fn get_opnd(&self, insn_id: InsnId) -> lir::Opnd { + self.opnds[insn_id.0].unwrap_or_else(|| panic!("Failed to get_opnd({insn_id})")) } /// Find or create a label for a given BlockId @@ -58,158 +85,203 @@ impl JITState { } } } - - /// Assume that this ISEQ doesn't escape EP. Return false if it's known to escape EP. - fn assume_no_ep_escape(iseq: IseqPtr) -> bool { - if iseq_escapes_ep(iseq) { - return false; - } - track_no_ep_escape_assumption(iseq); - true - } } -/// CRuby API to compile a given ISEQ +/// CRuby API to compile a given ISEQ. +/// If jit_exception is true, compile JIT code for handling exceptions. +/// See jit_compile_exception() for details. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, _ec: EcPtr) -> *const u8 { - // Do not test the JIT code in HIR tests - if cfg!(test) { - return std::ptr::null(); - } - +pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, jit_exception: bool) -> *const u8 { // Take a lock to avoid writing to ISEQ in parallel with Ractors. // with_vm_lock() does nothing if the program doesn't use Ractors. - let code_ptr = with_vm_lock(src_loc!(), || { - gen_iseq_entry_point(iseq) - }); + with_vm_lock(src_loc!(), || { + let cb = ZJITState::get_code_block(); + let mut code_ptr = with_time_stat(compile_time_ns, || gen_iseq_entry_point(cb, iseq, jit_exception)); + + if let Err(err) = &code_ptr { + // Assert that the ISEQ compiles if RubyVM::ZJIT.assert_compiles is enabled. + // We assert only `jit_exception: false` cases until we support exception handlers. + if ZJITState::assert_compiles_enabled() && !jit_exception { + let iseq_location = iseq_get_location(iseq, 0); + panic!("Failed to compile: {iseq_location}"); + } - // Assert that the ISEQ compiles if RubyVM::ZJIT.assert_compiles is enabled - if ZJITState::assert_compiles_enabled() && code_ptr.is_null() { - let iseq_location = iseq_get_location(iseq, 0); - panic!("Failed to compile: {iseq_location}"); - } + // For --zjit-stats, generate an entry that just increments exit_compilation_failure and exits + if get_option!(stats) { + code_ptr = gen_compile_error_counter(cb, err); + } + } + + // Always mark the code region executable if asm.compile() has been used. + // We need to do this even if code_ptr is None because gen_iseq() may have already used asm.compile(). + cb.mark_all_executable(); - code_ptr + code_ptr.map_or(std::ptr::null(), |ptr| ptr.raw_ptr(cb)) + }) } /// Compile an entry point for a given ISEQ -fn gen_iseq_entry_point(iseq: IseqPtr) -> *const u8 { +fn gen_iseq_entry_point(cb: &mut CodeBlock, iseq: IseqPtr, jit_exception: bool) -> Result<CodePtr, CompileError> { + // We don't support exception handlers yet + if jit_exception { + return Err(CompileError::ExceptionHandler); + } + // Compile ISEQ into High-level IR - let function = match compile_iseq(iseq) { - Some(function) => function, - None => return std::ptr::null(), - }; + let function = compile_iseq(iseq).inspect_err(|_| { + incr_counter!(failed_iseq_count); + })?; // Compile the High-level IR - let cb = ZJITState::get_code_block(); - let (start_ptr, mut branch_iseqs) = match gen_function(cb, iseq, &function) { - Some((start_ptr, branch_iseqs)) => { - // Remember the block address to reuse it later - let payload = get_or_create_iseq_payload(iseq); - payload.start_ptr = Some(start_ptr); - - // Compile an entry point to the JIT code - (gen_entry(cb, iseq, &function, start_ptr), branch_iseqs) - }, - None => (None, vec![]), - }; - - // Recursively compile callee ISEQs - while let Some((branch, iseq)) = branch_iseqs.pop() { - // Disable profiling. This will be the last use of the profiling information for the ISEQ. - unsafe { rb_zjit_profile_disable(iseq); } + let IseqCodePtrs { start_ptr, .. } = gen_iseq(cb, iseq, Some(&function)).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq, 0)); + })?; - // Compile the ISEQ - if let Some((callee_ptr, callee_branch_iseqs)) = gen_iseq(cb, iseq) { - let callee_addr = callee_ptr.raw_ptr(cb); - branch.regenerate(cb, |asm| { - asm.ccall(callee_addr, vec![]); - }); - branch_iseqs.extend(callee_branch_iseqs); - } else { - // Failed to compile the callee. Bail out of compiling this graph of ISEQs. - return std::ptr::null(); - } - } + Ok(start_ptr) +} - // Always mark the code region executable if asm.compile() has been used - cb.mark_all_executable(); +/// Stub a branch for a JIT-to-JIT call +pub fn gen_iseq_call(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<(), CompileError> { + // Compile a function stub + let stub_ptr = gen_function_stub(cb, iseq_call.clone()).inspect_err(|err| { + debug!("{err:?}: gen_function_stub failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + + // Update the JIT-to-JIT call to call the stub + let stub_addr = stub_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call function stub: {}", iseq_get_location(iseq, 0)); + asm.ccall(stub_addr, vec![]); + }); + Ok(()) +} - // Return a JIT code address or a null pointer - start_ptr.map(|start_ptr| start_ptr.raw_ptr(cb)).unwrap_or(std::ptr::null()) +/// Write an entry to the perf map in /tmp +fn register_with_perf(iseq_name: String, start_ptr: usize, code_size: usize) { + use std::io::Write; + let perf_map = format!("/tmp/perf-{}.map", std::process::id()); + let Ok(file) = std::fs::OpenOptions::new().create(true).append(true).open(&perf_map) else { + debug!("Failed to open perf map file: {perf_map}"); + return; + }; + let mut file = std::io::BufWriter::new(file); + let Ok(_) = writeln!(file, "{start_ptr:#x} {code_size:#x} zjit::{iseq_name}") else { + debug!("Failed to write {iseq_name} to perf map file: {perf_map}"); + return; + }; } -/// Compile a JIT entry -fn gen_entry(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function, function_ptr: CodePtr) -> Option<CodePtr> { +/// Compile a shared JIT entry trampoline +pub fn gen_entry_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { // Set up registers for CFP, EC, SP, and basic block arguments let mut asm = Assembler::new(); - gen_entry_prologue(&mut asm, iseq); - gen_method_params(&mut asm, iseq, function.block(BlockId(0))); + gen_entry_prologue(&mut asm); - // Jump to the first block using a call instruction - asm.ccall(function_ptr.raw_ptr(cb) as *const u8, vec![]); + // Jump to the first block using a call instruction. This trampoline is used + // as rb_zjit_func_t in jit_exec(), which takes (EC, CFP, rb_jit_func_t). + // So C_ARG_OPNDS[2] is rb_jit_func_t, which is (EC, CFP) -> VALUE. + asm.ccall_reg(C_ARG_OPNDS[2], VALUE_BITS); // Restore registers for CFP, EC, and SP after use - asm_comment!(asm, "exit to the interpreter"); - // On x86_64, maintain 16-byte stack alignment - if cfg!(target_arch = "x86_64") { - asm.cpop_into(SP); - } - asm.cpop_into(SP); - asm.cpop_into(EC); - asm.cpop_into(CFP); - asm.frame_teardown(); + asm_comment!(asm, "return to the interpreter"); + asm.frame_teardown(lir::JIT_PRESERVED_REGS); asm.cret(C_RET_OPND); - asm.compile(cb).map(|(start_ptr, _)| start_ptr) + let (code_ptr, gc_offsets) = asm.compile(cb)?; + assert!(gc_offsets.is_empty()); + if get_option!(perf) { + let start_ptr = code_ptr.raw_addr(cb); + let end_ptr = cb.get_write_ptr().raw_addr(cb); + let code_size = end_ptr - start_ptr; + register_with_perf("ZJIT entry trampoline".into(), start_ptr, code_size); + } + Ok(code_ptr) } -/// Compile an ISEQ into machine code -fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<(CodePtr, Vec<(Rc<Branch>, IseqPtr)>)> { +/// Compile an ISEQ into machine code if not compiled yet +fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { // Return an existing pointer if it's already compiled let payload = get_or_create_iseq_payload(iseq); - if let Some(start_ptr) = payload.start_ptr { - return Some((start_ptr, vec![])); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + match last_status { + Some(IseqStatus::Compiled(code_ptrs)) => return Ok(code_ptrs.clone()), + Some(IseqStatus::CantCompile(err)) => return Err(err.clone()), + _ => {}, + } + // If the ISEQ already hax MAX_ISEQ_VERSIONS, do not compile a new version. + if payload.versions.len() == MAX_ISEQ_VERSIONS { + return Err(CompileError::IseqVersionLimitReached); + } + + // Compile the ISEQ + let mut version = IseqVersion::new(iseq); + let code_ptrs = gen_iseq_body(cb, iseq, version, function); + match &code_ptrs { + Ok(code_ptrs) => { + unsafe { version.as_mut() }.status = IseqStatus::Compiled(code_ptrs.clone()); + incr_counter!(compiled_iseq_count); + } + Err(err) => { + unsafe { version.as_mut() }.status = IseqStatus::CantCompile(err.clone()); + incr_counter!(failed_iseq_count); + } + } + payload.versions.push(version); + code_ptrs +} + +/// Compile an ISEQ into machine code +fn gen_iseq_body(cb: &mut CodeBlock, iseq: IseqPtr, mut version: IseqVersionRef, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { + // If we ran out of code region, we shouldn't attempt to generate new code. + if cb.has_dropped_bytes() { + return Err(CompileError::OutOfMemory); } - // Convert ISEQ into High-level IR and optimize HIR - let function = match compile_iseq(iseq) { + // Convert ISEQ into optimized High-level IR if not given + let function = match function { Some(function) => function, - None => return None, + None => &compile_iseq(iseq)?, }; // Compile the High-level IR - let result = gen_function(cb, iseq, &function); - if let Some((start_ptr, _)) = result { - payload.start_ptr = Some(start_ptr); + let (iseq_code_ptrs, gc_offsets, iseq_calls) = gen_function(cb, iseq, version, function)?; + + // Stub callee ISEQs for JIT-to-JIT calls + for iseq_call in iseq_calls.iter() { + gen_iseq_call(cb, iseq_call)?; } - result + + // Prepare for GC + unsafe { version.as_mut() }.outgoing.extend(iseq_calls); + append_gc_offsets(iseq, version, &gc_offsets); + Ok(iseq_code_ptrs) } /// Compile a function -fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function) -> Option<(CodePtr, Vec<(Rc<Branch>, IseqPtr)>)> { - let mut jit = JITState::new(iseq, function.num_insns(), function.num_blocks()); - let mut asm = Assembler::new(); +fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, function: &Function) -> Result<(IseqCodePtrs, Vec<CodePtr>, Vec<IseqCallRef>), CompileError> { + let num_spilled_params = max_num_params(function).saturating_sub(ALLOC_REGS.len()); + let mut jit = JITState::new(iseq, version, function.num_insns(), function.num_blocks()); + let mut asm = Assembler::new_with_stack_slots(num_spilled_params); // Compile each basic block let reverse_post_order = function.rpo(); for &block_id in reverse_post_order.iter() { - let block = function.block(block_id); - asm_comment!(asm, "Block: {block_id}({})", block.params().map(|param| format!("{param}")).collect::<Vec<_>>().join(", ")); - // Write a label to jump to the basic block let label = jit.get_label(&mut asm, block_id); asm.write_label(label); - // Set up the frame at the first block - if block_id == BlockId(0) { - asm.frame_setup(); - } + let block = function.block(block_id); + asm_comment!( + asm, "{block_id}({}): {}", + block.params().map(|param| format!("{param}")).collect::<Vec<_>>().join(", "), + iseq_get_location(iseq, block.insn_idx), + ); // Compile all parameters - for &insn_id in block.params() { + for (idx, &insn_id) in block.params().enumerate() { match function.find(insn_id) { - Insn::Param { idx } => { + Insn::Param => { jit.opnds[insn_id.0] = Some(gen_param(&mut asm, idx)); }, insn => unreachable!("Non-param insn found in block.params: {insn:?}"), @@ -219,27 +291,66 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function) -> Optio // Compile all instructions for &insn_id in block.insns() { let insn = function.find(insn_id); - if gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn).is_none() { - debug!("Failed to compile insn: {insn_id} {insn:?}"); - return None; - } + if let Err(last_snapshot) = gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn) { + debug!("ZJIT: gen_function: Failed to compile insn: {insn_id} {insn}. Generating side-exit."); + gen_incr_counter(&mut asm, exit_counter_for_unhandled_hir_insn(&insn)); + gen_side_exit(&mut jit, &mut asm, &SideExitReason::UnhandledHIRInsn(insn_id), &function.frame_state(last_snapshot)); + // Don't bother generating code after a side-exit. We won't run it. + // TODO(max): Generate ud2 or equivalent. + break; + }; + // It's fine; we generated the instruction } + // Make sure the last patch point has enough space to insert a jump + asm.pad_patch_point(); } - if get_option!(dump_lir) { - println!("LIR:\nfn {}:\n{:?}", iseq_name(iseq), asm); + // Generate code if everything can be compiled + let result = asm.compile(cb); + if let Ok((start_ptr, _)) = result { + if get_option!(perf) { + let start_usize = start_ptr.raw_addr(cb); + let end_usize = cb.get_write_ptr().raw_addr(cb); + let code_size = end_usize - start_usize; + let iseq_name = iseq_get_location(iseq, 0); + register_with_perf(iseq_name, start_usize, code_size); + } + if ZJITState::should_log_compiled_iseqs() { + let iseq_name = iseq_get_location(iseq, 0); + ZJITState::log_compile(iseq_name); + } } + result.map(|(start_ptr, gc_offsets)| { + // Make sure jit_entry_ptrs can be used as a parallel vector to jit_entry_insns() + jit.jit_entries.sort_by_key(|jit_entry| jit_entry.borrow().jit_entry_idx); - // Generate code if everything can be compiled - asm.compile(cb).map(|(start_ptr, _)| (start_ptr, jit.branch_iseqs)) + let jit_entry_ptrs = jit.jit_entries.iter().map(|jit_entry| + jit_entry.borrow().start_addr.get().expect("start_addr should have been set by pos_marker in gen_entry_point") + ).collect(); + (IseqCodePtrs { start_ptr, jit_entry_ptrs }, gc_offsets, jit.iseq_calls) + }) } /// Compile an instruction -fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, function: &Function, insn_id: InsnId, insn: &Insn) -> Option<()> { +fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, function: &Function, insn_id: InsnId, insn: &Insn) -> Result<(), InsnId> { // Convert InsnId to lir::Opnd macro_rules! opnd { ($insn_id:ident) => { - jit.get_opnd(*$insn_id)? + jit.get_opnd($insn_id.clone()) + }; + } + + macro_rules! opnds { + ($insn_ids:ident) => { + { + $insn_ids.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect::<Vec<_>>() + } + }; + } + + macro_rules! no_output { + ($call:expr) => { + { let () = $call; return Ok(()); } }; } @@ -248,160 +359,895 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } let out_opnd = match insn { - Insn::PutSelf => gen_putself(), - Insn::Const { val: Const::Value(val) } => gen_const(*val), - Insn::NewArray { elements, state } => gen_new_array(jit, asm, elements, &function.frame_state(*state)), + &Insn::Const { val: Const::Value(val) } => gen_const_value(val), + &Insn::Const { val: Const::CPtr(val) } => gen_const_cptr(val), + &Insn::Const { val: Const::CInt64(val) } => gen_const_long(val), + &Insn::Const { val: Const::CUInt16(val) } => gen_const_uint16(val), + &Insn::Const { val: Const::CUInt32(val) } => gen_const_uint32(val), + &Insn::Const { val: Const::CShape(val) } => { + assert_eq!(SHAPE_ID_NUM_BITS, 32); + gen_const_uint32(val.0) + } + Insn::Const { .. } => panic!("Unexpected Const in gen_insn: {insn}"), + Insn::NewArray { elements, state } => gen_new_array(asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewHash { elements, state } => gen_new_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::ArrayDup { val, state } => gen_array_dup(asm, opnd!(val), &function.frame_state(*state)), - Insn::Param { idx } => unreachable!("block.insns should not have Insn::Param({idx})"), - Insn::Snapshot { .. } => return Some(()), // we don't need to do anything for this instruction at the moment - Insn::Jump(branch) => return gen_jump(jit, asm, branch), - Insn::IfTrue { val, target } => return gen_if_true(jit, asm, opnd!(val), target), - Insn::IfFalse { val, target } => return gen_if_false(jit, asm, opnd!(val), target), - Insn::SendWithoutBlock { call_info, cd, state, .. } => gen_send_without_block(jit, asm, call_info, *cd, &function.frame_state(*state))?, - Insn::SendWithoutBlockDirect { iseq, self_val, args, .. } => gen_send_without_block_direct(cb, jit, asm, *iseq, opnd!(self_val), args)?, - Insn::Return { val } => return Some(gen_return(asm, opnd!(val))?), - Insn::FixnumAdd { left, right, state } => gen_fixnum_add(asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumSub { left, right, state } => gen_fixnum_sub(asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumMult { left, right, state } => gen_fixnum_mult(asm, opnd!(left), opnd!(right), &function.frame_state(*state))?, - Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right))?, - Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right))?, - Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right))?, - Insn::FixnumLe { left, right } => gen_fixnum_le(asm, opnd!(left), opnd!(right))?, - Insn::FixnumGt { left, right } => gen_fixnum_gt(asm, opnd!(left), opnd!(right))?, - Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right))?, - Insn::Test { val } => gen_test(asm, opnd!(val))?, - Insn::GuardType { val, guard_type, state } => gen_guard_type(asm, opnd!(val), *guard_type, &function.frame_state(*state))?, - Insn::GuardBitEquals { val, expected, state } => gen_guard_bit_equals(asm, opnd!(val), *expected, &function.frame_state(*state))?, - Insn::PatchPoint(_) => return Some(()), // For now, rb_zjit_bop_redefined() panics. TODO: leave a patch point and fix rb_zjit_bop_redefined() - Insn::CCall { cfun, args, name: _, return_type: _, elidable: _ } => gen_ccall(jit, asm, *cfun, args)?, - _ => { - debug!("ZJIT: gen_function: unexpected insn {:?}", insn); - return None; + Insn::ArrayAref { array, index, .. } => gen_array_aref(asm, opnd!(array), opnd!(index)), + Insn::ArrayAset { array, index, val } => { + no_output!(gen_array_aset(asm, opnd!(array), opnd!(index), opnd!(val))) } + Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), + Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), + Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, opnd!(val), &function.frame_state(*state)), + &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(asm, class, &function.frame_state(state)), + Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), + // concatstrings shouldn't have 0 strings + // If it happens we abort the compilation for now + Insn::StringConcat { strings, state, .. } if strings.is_empty() => return Err(*state), + Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)), + &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), + Insn::StringSetbyteFixnum { string, index, value } => gen_string_setbyte_fixnum(asm, opnd!(string), opnd!(index), opnd!(value)), + Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)), + Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), + Insn::Param => unreachable!("block.insns should not have Insn::Param"), + Insn::Snapshot { .. } => return Ok(()), // we don't need to do anything for this instruction at the moment + Insn::Jump(branch) => no_output!(gen_jump(jit, asm, branch)), + Insn::IfTrue { val, target } => no_output!(gen_if_true(jit, asm, opnd!(val), target)), + Insn::IfFalse { val, target } => no_output!(gen_if_false(jit, asm, opnd!(val), target)), + &Insn::Send { cd, blockiseq, state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::SendWithoutBlock { cd, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), + Insn::SendWithoutBlockDirect { cme, iseq, recv, args, state, .. } => gen_send_iseq_direct(cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state), None), + &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), + Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), + // Ensure we have enough room fit ec, self, and arguments + // TODO remove this check when we have stack args (we can use Time.new to test it) + Insn::InvokeBuiltin { bf, state, .. } if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) => return Err(*state), + Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, *leaf, opnds!(args)), + &Insn::EntryPoint { jit_entry_idx } => no_output!(gen_entry_point(jit, asm, jit_entry_idx)), + Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), + Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right)), + Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right)), + Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right)), + Insn::FixnumLe { left, right } => gen_fixnum_le(asm, opnd!(left), opnd!(right)), + Insn::FixnumGt { left, right } => gen_fixnum_gt(asm, opnd!(left), opnd!(right)), + Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right)), + Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right)), + Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right)), + Insn::FixnumXor { left, right } => gen_fixnum_xor(asm, opnd!(left), opnd!(right)), + &Insn::FixnumLShift { left, right, state } => { + // We only create FixnumLShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_lshift(jit, asm, opnd!(left), shift_amount, &function.frame_state(state)) + } + &Insn::FixnumRShift { left, right } => { + // We only create FixnumRShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_rshift(asm, opnd!(left), shift_amount) + } + &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::FixnumAref { recv, index } => gen_fixnum_aref(asm, opnd!(recv), opnd!(index)), + Insn::IsNil { val } => gen_isnil(asm, opnd!(val)), + &Insn::IsMethodCfunc { val, cd, cfunc, state: _ } => gen_is_method_cfunc(jit, asm, opnd!(val), cd, cfunc), + &Insn::IsBitEqual { left, right } => gen_is_bit_equal(asm, opnd!(left), opnd!(right)), + &Insn::IsBitNotEqual { left, right } => gen_is_bit_not_equal(asm, opnd!(left), opnd!(right)), + &Insn::BoxBool { val } => gen_box_bool(asm, opnd!(val)), + &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, opnd!(val), &function.frame_state(state)), + &Insn::UnboxFixnum { val } => gen_unbox_fixnum(asm, opnd!(val)), + Insn::Test { val } => gen_test(asm, opnd!(val)), + Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), + Insn::GuardTypeNot { val, guard_type, state } => gen_guard_type_not(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), + &Insn::GuardBitEquals { val, expected, reason, state } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, reason, &function.frame_state(state)), + &Insn::GuardBlockParamProxy { level, state } => no_output!(gen_guard_block_param_proxy(jit, asm, level, &function.frame_state(state))), + Insn::GuardNotFrozen { recv, state } => gen_guard_not_frozen(jit, asm, opnd!(recv), &function.frame_state(*state)), + Insn::GuardNotShared { recv, state } => gen_guard_not_shared(jit, asm, opnd!(recv), &function.frame_state(*state)), + &Insn::GuardLess { left, right, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::GuardGreaterEq { left, right, state } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::GuardSuperMethodEntry { cme, state } => no_output!(gen_guard_super_method_entry(jit, asm, cme, &function.frame_state(state))), + Insn::GetBlockHandler => gen_get_block_handler(jit, asm), + Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), + Insn::CCall { cfunc, recv, args, name, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), + // Give up CCallWithFrame for 7+ args since asm.ccall() supports at most 6 args (recv + args). + // There's no test case for this because no core cfuncs have this many parameters. But C extensions could have such methods. + Insn::CCallWithFrame { cd, state, args, .. } if args.len() + 1 > C_ARG_OPNDS.len() => + gen_send_without_block(jit, asm, *cd, &function.frame_state(*state), SendFallbackReason::CCallWithFrameTooManyArgs), + Insn::CCallWithFrame { cfunc, recv, name, args, cme, state, blockiseq, .. } => + gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *blockiseq, &function.frame_state(*state)), + Insn::CCallVariadic { cfunc, recv, name, args, cme, state, blockiseq, return_type: _, elidable: _ } => { + gen_ccall_variadic(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *blockiseq, &function.frame_state(*state)) + } + Insn::GetIvar { self_val, id, ic, state: _ } => gen_getivar(jit, asm, opnd!(self_val), *id, *ic), + Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), + Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), + &Insn::GetLocal { ep_offset, level, use_sp, .. } => gen_getlocal(asm, ep_offset, level, use_sp), + &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), + Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), + Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, *id, *ic, &function.frame_state(*state)), + Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, *id, opnd!(val), *ic, &function.frame_state(*state))), + Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), + Insn::FixnumBitCheck { val, index } => gen_fixnum_bit_check(asm, opnd!(val), *index), + Insn::SideExit { state, reason } => no_output!(gen_side_exit(jit, asm, reason, &function.frame_state(*state))), + Insn::PutSpecialObject { value_type } => gen_putspecialobject(asm, *value_type), + Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)), + Insn::Defined { op_type, obj, pushval, v, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), &function.frame_state(*state)), + Insn::GetSpecialSymbol { symbol_type, state: _ } => gen_getspecial_symbol(asm, *symbol_type), + Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)), + &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)), + Insn::IncrCounterPtr { counter_ptr } => no_output!(gen_incr_counter_ptr(asm, *counter_ptr)), + Insn::ObjToString { val, cd, state, .. } => gen_objtostring(jit, asm, opnd!(val), *cd, &function.frame_state(*state)), + &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, &function.frame_state(state))), + &Insn::HashDup { val, state } => { gen_hash_dup(asm, opnd!(val), &function.frame_state(state)) }, + &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, opnd!(hash), opnd!(key), &function.frame_state(state)) }, + &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, + &Insn::ArrayPush { array, val, state } => { no_output!(gen_array_push(asm, opnd!(array), opnd!(val), &function.frame_state(state))) }, + &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::ToArray { val, state } => { gen_to_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::DefinedIvar { self_val, id, pushval, .. } => { gen_defined_ivar(asm, opnd!(self_val), id, pushval) }, + &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state))) }, + &Insn::GuardShape { val, shape, state } => gen_guard_shape(jit, asm, opnd!(val), shape, &function.frame_state(state)), + Insn::LoadPC => gen_load_pc(asm), + Insn::LoadEC => gen_load_ec(), + Insn::LoadSelf => gen_load_self(), + &Insn::LoadField { recv, id, offset, return_type } => gen_load_field(asm, opnd!(recv), id, offset, return_type), + &Insn::StoreField { recv, id, offset, val } => no_output!(gen_store_field(asm, opnd!(recv), id, offset, opnd!(val), function.type_of(val))), + &Insn::WriteBarrier { recv, val } => no_output!(gen_write_barrier(asm, opnd!(recv), opnd!(val), function.type_of(val))), + &Insn::IsBlockGiven => gen_is_block_given(jit, asm), + Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, opnds!(elements), opnd!(target), &function.frame_state(*state)), + Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, opnds!(elements), opnd!(fmt), opnd!(buffer), &function.frame_state(*state)), + &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, ary, opnd!(target), &function.frame_state(state)), + Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + &Insn::IsA { val, class } => gen_is_a(asm, opnd!(val), opnd!(class)), + &Insn::ArrayMax { state, .. } + | &Insn::Throw { state, .. } + => return Err(state), }; + assert!(insn.has_output(), "Cannot write LIR output of HIR instruction with no output: {insn}"); + // If the instruction has an output, remember it in jit.opnds jit.opnds[insn_id.0] = Some(out_opnd); - Some(()) + Ok(()) } -/// Lowering for [`Insn::CCall`]. This is a low-level raw call that doesn't know -/// anything about the callee, so handling for e.g. GC safety is dealt with elsewhere. -fn gen_ccall(jit: &mut JITState, asm: &mut Assembler, cfun: *const u8, args: &[InsnId]) -> Option<lir::Opnd> { - let mut lir_args = Vec::with_capacity(args.len()); - for &arg in args { - lir_args.push(jit.get_opnd(arg)?); +/// Gets the EP of the ISeq of the containing method, or "local level". +/// Equivalent of GET_LEP() macro. +fn gen_get_lep(jit: &JITState, asm: &mut Assembler) -> Opnd { + // Equivalent of get_lvar_level() in compile.c + fn get_lvar_level(mut iseq: IseqPtr) -> u32 { + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; + let mut level = 0; + while iseq != local_iseq { + iseq = unsafe { rb_get_iseq_body_parent_iseq(iseq) }; + level += 1; + } + + level } - Some(asm.ccall(cfun, lir_args)) + let level = get_lvar_level(jit.iseq); + gen_get_ep(asm, level) } -/// Compile an interpreter entry block to be inserted into an ISEQ -fn gen_entry_prologue(asm: &mut Assembler, iseq: IseqPtr) { - asm_comment!(asm, "ZJIT entry point: {}", iseq_get_location(iseq, 0)); - asm.frame_setup(); +// Get EP at `level` from CFP +fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { + // Load environment pointer EP from CFP into a register + let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); + let mut ep_opnd = asm.load(ep_opnd); + + for _ in 0..level { + // Get the previous EP from the current EP + // See GET_PREV_EP(ep) macro + // VALUE *prev_ep = ((VALUE *)((ep)[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03)) + const UNTAGGING_MASK: Opnd = Opnd::Imm(!0x03); + let offset = SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL; + ep_opnd = asm.load(Opnd::mem(64, ep_opnd, offset)); + ep_opnd = asm.and(ep_opnd, UNTAGGING_MASK); + } - // Save the registers we'll use for CFP, EP, SP - asm.cpush(CFP); - asm.cpush(EC); - asm.cpush(SP); - // On x86_64, maintain 16-byte stack alignment - if cfg!(target_arch = "x86_64") { - asm.cpush(SP); + ep_opnd +} + +fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *const rb_call_data, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + // TODO: Specialize for immediate types + // Call rb_vm_objtostring(iseq, recv, cd) + let ret = asm_ccall!(asm, rb_vm_objtostring, VALUE::from(jit.iseq).into(), val, Opnd::const_ptr(cd)); + + // TODO: Call `to_s` on the receiver if rb_vm_objtostring returns Qundef + // Need to replicate what CALL_SIMPLE_METHOD does + asm_comment!(asm, "side-exit if rb_vm_objtostring returns Qundef"); + asm.cmp(ret, Qundef.into()); + asm.je(side_exit(jit, state, ObjToStringFallback)); + + ret +} + +fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, state: &FrameState) -> Opnd { + match op_type as defined_type { + DEFINED_YIELD => { + // `yield` goes to the block handler stowed in the "local" iseq which is + // the current iseq or a parent. Only the "method" iseq type can be passed a + // block handler. (e.g. `yield` in the top level script is a syntax error.) + // + // Similar to gen_is_block_given + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(jit.iseq) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + let lep = gen_get_lep(jit, asm); + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + let pushval = asm.load(pushval.into()); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qnil.into(), pushval) + } else { + Qnil.into() + } + } + _ => { + // Save the PC and SP because the callee may allocate or call #respond_to? + gen_prepare_non_leaf_call(jit, asm, state); + + // TODO: Inline the cases for each op_type + // Call vm_defined(ec, reg_cfp, op_type, obj, v) + let def_result = asm_ccall!(asm, rb_vm_defined, EC, CFP, op_type.into(), obj.into(), tested_value); + + asm.cmp(def_result.with_num_bits(8), 0.into()); + asm.csel_ne(pushval.into(), Qnil.into()) + } } +} - // EC and CFP are pased as arguments - asm.mov(EC, C_ARG_OPNDS[0]); - asm.mov(CFP, C_ARG_OPNDS[1]); +/// Similar to gen_defined for DEFINED_YIELD +fn gen_is_block_given(jit: &JITState, asm: &mut Assembler) -> Opnd { + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(jit.iseq) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + let lep = gen_get_lep(jit, asm); + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qfalse.into(), Qtrue.into()) + } else { + Qfalse.into() + } +} - // Load the current SP from the CFP into REG_SP - asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); +fn gen_unbox_fixnum(asm: &mut Assembler, val: Opnd) -> Opnd { + asm.rshift(val, Opnd::UImm(1)) +} - // TODO: Support entry chain guard when ISEQ has_opt +/// Get a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. +/// We generate this instruction with level=0 only when the local variable is on the heap, so we +/// can't optimize the level=0 case using the SP register. +fn gen_getlocal(asm: &mut Assembler, local_ep_offset: u32, level: u32, use_sp: bool) -> lir::Opnd { + let local_ep_offset = i32::try_from(local_ep_offset).unwrap_or_else(|_| panic!("Could not convert local_ep_offset {local_ep_offset} to i32")); + if level > 0 { + gen_incr_counter(asm, Counter::vm_read_from_parent_iseq_local_count); + } + let local = if use_sp { + assert_eq!(level, 0, "use_sp optimization should be used only for level=0 locals"); + let offset = -(SIZEOF_VALUE_I32 * (local_ep_offset + 1)); + Opnd::mem(64, SP, offset) + } else { + let ep = gen_get_ep(asm, level); + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + Opnd::mem(64, ep, offset) + }; + asm.load(local) } -/// Assign method arguments to basic block arguments at JIT entry -fn gen_method_params(asm: &mut Assembler, iseq: IseqPtr, entry_block: &Block) { - let num_params = entry_block.params().len(); - if num_params > 0 { - asm_comment!(asm, "set method params: {num_params}"); +/// Set a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. +/// We generate this instruction with level=0 only when the local variable is on the heap, so we +/// can't optimize the level=0 case using the SP register. +fn gen_setlocal(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: u32, level: u32) { + let local_ep_offset = c_int::try_from(local_ep_offset).unwrap_or_else(|_| panic!("Could not convert local_ep_offset {local_ep_offset} to i32")); + if level > 0 { + gen_incr_counter(asm, Counter::vm_write_to_parent_iseq_local_count); + } + let ep = gen_get_ep(asm, level); - // Allocate registers for basic block arguments - let params: Vec<Opnd> = (0..num_params).map(|idx| - gen_param(asm, idx) - ).collect(); + // When we've proved that we're writing an immediate, + // we can skip the write barrier. + if val_type.is_immediate() { + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + asm.mov(Opnd::mem(64, ep, offset), val); + } else { + // We're potentially writing a reference to an IMEMO/env object, + // so take care of the write barrier with a function. + let local_index = -local_ep_offset; + asm_ccall!(asm, rb_vm_env_write, ep, local_index.into(), val); + } +} + +fn gen_guard_block_param_proxy(jit: &JITState, asm: &mut Assembler, level: u32, state: &FrameState) { + // Bail out if the `&block` local variable has been modified + let ep = gen_get_ep(asm, level); + let flags = Opnd::mem(64, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + asm.test(flags, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.jnz(side_exit(jit, state, SideExitReason::BlockParamProxyModified)); + + // This handles two cases which are nearly identical + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + + // Bail ouf if the block handler is neither ISEQ nor ifunc + let block_handler = asm.load(Opnd::mem(64, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + asm.test(block_handler, 0x1.into()); + asm.jz(side_exit(jit, state, SideExitReason::BlockParamProxyNotIseqOrIfunc)); +} + +fn gen_guard_not_frozen(jit: &JITState, asm: &mut Assembler, recv: Opnd, state: &FrameState) -> Opnd { + let recv = asm.load(recv); + // It's a heap object, so check the frozen flag + let flags = asm.load(Opnd::mem(64, recv, RUBY_OFFSET_RBASIC_FLAGS)); + asm.test(flags, (RUBY_FL_FREEZE as u64).into()); + // Side-exit if frozen + asm.jnz(side_exit(jit, state, GuardNotFrozen)); + recv +} + +fn gen_guard_not_shared(jit: &JITState, asm: &mut Assembler, recv: Opnd, state: &FrameState) -> Opnd { + let recv = asm.load(recv); + // It's a heap object, so check the shared flag + let flags = asm.load(Opnd::mem(VALUE_BITS, recv, RUBY_OFFSET_RBASIC_FLAGS)); + asm.test(flags, (RUBY_ELTS_SHARED as u64).into()); + asm.jnz(side_exit(jit, state, SideExitReason::GuardNotShared)); + recv +} + +fn gen_guard_less(jit: &JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jge(side_exit(jit, state, SideExitReason::GuardLess)); + left +} + +fn gen_guard_greater_eq(jit: &JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jl(side_exit(jit, state, SideExitReason::GuardGreaterEq)); + left +} + +/// Guard that the method entry at ep[VM_ENV_DATA_INDEX_ME_CREF] matches the expected CME. +/// This ensures we're calling super from the expected method context. +fn gen_guard_super_method_entry( + jit: &JITState, + asm: &mut Assembler, + cme: *const rb_callable_method_entry_t, + state: &FrameState, +) { + asm_comment!(asm, "guard super method entry"); + let lep = gen_get_lep(jit, asm); + let ep_me_opnd = Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF); + let ep_me = asm.load(ep_me_opnd); + asm.cmp(ep_me, Opnd::UImm(cme as u64)); + asm.jne(side_exit(jit, state, SideExitReason::GuardSuperMethodEntry)); +} + +/// Get the block handler from ep[VM_ENV_DATA_INDEX_SPECVAL] at the local EP (LEP). +fn gen_get_block_handler(jit: &JITState, asm: &mut Assembler) -> Opnd { + asm_comment!(asm, "get block handler from LEP"); + let lep = gen_get_lep(jit, asm); + asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)) +} - // Assign local variables to the basic block arguments - for (idx, ¶m) in params.iter().enumerate() { - let local = gen_getlocal(asm, iseq, idx); - asm.load_into(param, local); +fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { + unsafe extern "C" { + fn rb_vm_opt_getconstant_path(ec: EcPtr, cfp: CfpPtr, ic: *const iseq_inline_constant_cache) -> VALUE; + } + + // Anything could be called on const_missing + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic)) +} + +fn gen_fixnum_bit_check(asm: &mut Assembler, val: Opnd, index: u8) -> Opnd { + let bit_test: u64 = 0x01 << (index + 1); + asm.test(val, bit_test.into()); + asm.csel_z(Qtrue.into(), Qfalse.into()) +} + +fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec<Opnd>) -> lir::Opnd { + assert!(bf.argc + 2 <= C_ARG_OPNDS.len() as i32, + "gen_invokebuiltin should not be called for builtin function {} with too many arguments: {}", + unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }, + bf.argc); + if leaf { + gen_prepare_leaf_call_with_gc(asm, state); + } else { + // Anything can happen inside builtin functions + gen_prepare_non_leaf_call(jit, asm, state); + } + + let mut cargs = vec![EC]; + cargs.extend(args); + + asm.count_call_to(unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }); + asm.ccall(bf.func_ptr as *const u8, cargs) +} + +/// Record a patch point that should be invalidated on a given invariant +fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { + let invariant = *invariant; + let exit = build_side_exit(jit, state); + + // Let compile_exits compile a side exit. Let scratch_split lower it with split_patch_point. + asm.patch_point(Target::SideExit { exit, reason: PatchPoint(invariant) }, invariant, jit.version); +} + +/// This is used by scratch_split to lower PatchPoint into PadPatchPoint and PosMarker. +/// It's called at scratch_split so that we can use the Label after side-exit deduplication in compile_exits. +pub fn split_patch_point(asm: &mut Assembler, target: &Target, invariant: Invariant, version: IseqVersionRef) { + let Target::Label(exit_label) = *target else { + unreachable!("PatchPoint's target should have been lowered to Target::Label by compile_exits: {target:?}"); + }; + + // Fill nop instructions if the last patch point is too close. + asm.pad_patch_point(); + + // Remember the current address as a patch point + asm.pos_marker(move |code_ptr, cb| { + let side_exit_ptr = cb.resolve_label(exit_label); + match invariant { + Invariant::BOPRedefined { klass, bop } => { + track_bop_assumption(klass, bop, code_ptr, side_exit_ptr, version); + } + Invariant::MethodRedefined { klass: _, method: _, cme } => { + track_cme_assumption(cme, code_ptr, side_exit_ptr, version); + } + Invariant::StableConstantNames { idlist } => { + track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr, version); + } + Invariant::NoTracePoint => { + track_no_trace_point_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoEPEscape(iseq) => { + track_no_ep_escape_assumption(iseq, code_ptr, side_exit_ptr, version); + } + Invariant::SingleRactorMode => { + track_single_ractor_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoSingletonClass { klass } => { + track_no_singleton_class_assumption(klass, code_ptr, side_exit_ptr, version); + } } + }); +} + +/// Generate code for a C function call that pushes a frame +fn gen_ccall_with_frame( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + blockiseq: Option<IseqPtr>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::non_variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + let caller_stack_size = state.stack().len() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (and block arguments if any) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(block_iseq) = blockiseq { + // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). + // VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. + // rb_captured_block->code.iseq aliases with cfp->block_code. + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(block_iseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + pc: PC_POISON, + specval: block_handler_specval, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, cfunc_args); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result +} + +/// Lowering for [`Insn::CCall`]. This is a low-level raw call that doesn't know +/// anything about the callee, so handling for e.g. GC safety is dealt with elsewhere. +fn gen_ccall(asm: &mut Assembler, cfunc: *const u8, name: ID, recv: Opnd, args: Vec<Opnd>) -> lir::Opnd { + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + asm.ccall(cfunc, cfunc_args) +} + +/// Generate code for a variadic C function call +/// func(int argc, VALUE *argv, VALUE recv) +fn gen_ccall_variadic( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + blockiseq: Option<IseqPtr>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + + // Compute the caller's stack size after consuming recv and args. + // state.stack() includes recv + args, so subtract both. + let caller_stack_size = state.stack_size() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (like gen_ccall_with_frame does) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(block_iseq) = blockiseq { + // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). + // VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. + // rb_captured_block->code.iseq aliases with cfp->block_code. + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(block_iseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + specval: block_handler_specval, + pc: PC_POISON, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let argv_ptr = gen_push_opnds(asm, &args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, vec![args.len().into(), argv_ptr, recv]); + gen_pop_opnds(asm, &args); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result +} + +/// Emit an uncached instance variable lookup +fn gen_getivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry) -> Opnd { + if ic.is_null() { + asm_ccall!(asm, rb_ivar_get, recv, id.0.into()) + } else { + let iseq = Opnd::Value(jit.iseq.into()); + asm_ccall!(asm, rb_vm_getinstancevariable, iseq, recv, id.0.into(), Opnd::const_ptr(ic)) } } -/// Set branch params to basic block arguments -fn gen_branch_params(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) -> Option<()> { - if !branch.args.is_empty() { - asm_comment!(asm, "set branch params: {}", branch.args.len()); - let mut moves: Vec<(Reg, Opnd)> = vec![]; - for (idx, &arg) in branch.args.iter().enumerate() { - moves.push((param_reg(idx), jit.get_opnd(arg)?)); +/// Emit an uncached instance variable store +fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { + // Setting an ivar can raise FrozenError, so we need proper frame state for exception handling. + gen_prepare_non_leaf_call(jit, asm, state); + if ic.is_null() { + asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); + } else { + let iseq = Opnd::Value(jit.iseq.into()); + asm_ccall!(asm, rb_vm_setinstancevariable, iseq, recv, id.0.into(), val, Opnd::const_ptr(ic)); + } +} + +fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_getclassvariable, VALUE::from(jit.iseq).into(), CFP, id.0.into(), Opnd::const_ptr(ic)) +} + +fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_setclassvariable, VALUE::from(jit.iseq).into(), CFP, id.0.into(), val, Opnd::const_ptr(ic)); +} + +/// Look up global variables +fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, state: &FrameState) -> Opnd { + // `Warning` module's method `warn` can be called when reading certain global variables + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_gvar_get, id.0.into()) +} + +/// Intern a string +fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_str_intern, val) +} + +/// Set global variables +fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, state: &FrameState) { + // When trace_var is used, setting a global variable can cause exceptions + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_gvar_set, id.0.into(), val); +} + +/// Side-exit into the interpreter +fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, state: &FrameState) { + asm.jmp(side_exit(jit, state, *reason)); +} + +/// Emit a special object lookup +fn gen_putspecialobject(asm: &mut Assembler, value_type: SpecialObjectType) -> Opnd { + // Get the EP of the current CFP and load it into a register + let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); + let ep_reg = asm.load(ep_opnd); + + asm_ccall!(asm, rb_vm_get_special_object, ep_reg, Opnd::UImm(u64::from(value_type))) +} + +fn gen_getspecial_symbol(asm: &mut Assembler, symbol_type: SpecialBackrefSymbol) -> Opnd { + // Fetch a "special" backref based on the symbol type + + let backref = asm_ccall!(asm, rb_backref_get,); + + match symbol_type { + SpecialBackrefSymbol::LastMatch => { + asm_ccall!(asm, rb_reg_last_match, backref) + } + SpecialBackrefSymbol::PreMatch => { + asm_ccall!(asm, rb_reg_match_pre, backref) + } + SpecialBackrefSymbol::PostMatch => { + asm_ccall!(asm, rb_reg_match_post, backref) + } + SpecialBackrefSymbol::LastGroup => { + asm_ccall!(asm, rb_reg_match_last, backref) } - asm.parallel_mov(moves); } - Some(()) } -/// Get the local variable at the given index -fn gen_getlocal(asm: &mut Assembler, iseq: IseqPtr, local_idx: usize) -> lir::Opnd { - let ep_offset = local_idx_to_ep_offset(iseq, local_idx); +fn gen_getspecial_number(asm: &mut Assembler, nth: u64, state: &FrameState) -> Opnd { + // Fetch the N-th match from the last backref based on type shifted by 1 - if JITState::assume_no_ep_escape(iseq) { - // Create a reference to the local variable using the SP register. We assume EP == BP. - // TODO: Implement the invalidation in rb_zjit_invalidate_ep_is_bp() - let offs = -(SIZEOF_VALUE_I32 * (ep_offset + 1)); - Opnd::mem(64, SP, offs) - } else { - // Get the EP of the current CFP - let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); - let ep_reg = asm.load(ep_opnd); + let backref = asm_ccall!(asm, rb_backref_get,); + + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_reg_nth_match, Opnd::Imm((nth >> 1).try_into().unwrap()), backref) +} + +fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameState) { + // Check for interrupts + // see RUBY_VM_CHECK_INTS(ec) macro + asm_comment!(asm, "RUBY_VM_CHECK_INTS(ec)"); + // Not checking interrupt_mask since it's zero outside finalize_deferred_heap_pages, + // signal_exec, or rb_postponed_job_flush. + let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32)); + asm.test(interrupt_flag, interrupt_flag); + asm.jnz(side_exit(jit, state, SideExitReason::Interrupt)); +} + +fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_hash_resurrect, val) +} + +fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aref, hash, key) +} + +fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aset, hash, key, val); +} + +fn gen_array_push(asm: &mut Assembler, array: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_push, array, val); +} + +fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qtrue), val) +} + +fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qfalse), val) +} + +fn gen_defined_ivar(asm: &mut Assembler, self_val: Opnd, id: ID, pushval: VALUE) -> lir::Opnd { + asm_ccall!(asm, rb_zjit_defined_ivar, self_val, id.0.into(), Opnd::Value(pushval)) +} + +fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_ary_concat, left, right); +} - // Create a reference to the local variable using cfp->ep - let offs = -(SIZEOF_VALUE_I32 * ep_offset); - Opnd::mem(64, ep_reg, offs) +fn gen_guard_shape(jit: &mut JITState, asm: &mut Assembler, val: Opnd, shape: ShapeId, state: &FrameState) -> Opnd { + gen_incr_counter(asm, Counter::guard_shape_count); + let shape_id_offset = unsafe { rb_shape_id_offset() }; + let val = asm.load(val); + let shape_opnd = Opnd::mem(SHAPE_ID_NUM_BITS as u8, val, shape_id_offset); + asm.cmp(shape_opnd, Opnd::UImm(shape.0 as u64)); + asm.jne(side_exit(jit, state, SideExitReason::GuardShape(shape))); + val +} + +fn gen_load_pc(asm: &mut Assembler) -> Opnd { + asm.load(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC)) +} + +fn gen_load_ec() -> Opnd { + EC +} + +fn gen_load_self() -> Opnd { + Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF) +} + +fn gen_load_field(asm: &mut Assembler, recv: Opnd, id: ID, offset: i32, return_type: Type) -> Opnd { + asm_comment!(asm, "Load field id={} offset={}", id.contents_lossy(), offset); + let recv = asm.load(recv); + asm.load(Opnd::mem(return_type.num_bits(), recv, offset)) +} + +fn gen_store_field(asm: &mut Assembler, recv: Opnd, id: ID, offset: i32, val: Opnd, val_type: Type) { + asm_comment!(asm, "Store field id={} offset={}", id.contents_lossy(), offset); + let recv = asm.load(recv); + asm.store(Opnd::mem(val_type.num_bits(), recv, offset), val); +} + +fn gen_write_barrier(asm: &mut Assembler, recv: Opnd, val: Opnd, val_type: Type) { + // See RB_OBJ_WRITE/rb_obj_write: it's just assignment and rb_obj_written(). + // rb_obj_written() does: if (!RB_SPECIAL_CONST_P(val)) { rb_gc_writebarrier(recv, val); } + if !val_type.is_immediate() { + asm_comment!(asm, "Write barrier"); + let recv = asm.load(recv); + asm_ccall!(asm, rb_zjit_writebarrier_check_immediate, recv, val); } } -/// Compile self in the current frame -fn gen_putself() -> lir::Opnd { - Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_SELF) +/// Compile an interpreter entry block to be inserted into an ISEQ +fn gen_entry_prologue(asm: &mut Assembler) { + asm_comment!(asm, "ZJIT entry trampoline"); + // Save the registers we'll use for CFP, EP, SP + asm.frame_setup(lir::JIT_PRESERVED_REGS); + + // EC and CFP are passed as arguments + asm.mov(EC, C_ARG_OPNDS[0]); + asm.mov(CFP, C_ARG_OPNDS[1]); + + // Load the current SP from the CFP into REG_SP + asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); +} + +/// Set branch params to basic block arguments +fn gen_branch_params(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { + if branch.args.is_empty() { + return; + } + + asm_comment!(asm, "set branch params: {}", branch.args.len()); + asm.parallel_mov(branch.args.iter().enumerate().map(|(idx, &arg)| + (param_opnd(idx), jit.get_opnd(arg)) + ).collect()); } /// Compile a constant -fn gen_const(val: VALUE) -> lir::Opnd { +fn gen_const_value(val: VALUE) -> lir::Opnd { // Just propagate the constant value and generate nothing Opnd::Value(val) } +/// Compile Const::CPtr +fn gen_const_cptr(val: *const u8) -> lir::Opnd { + Opnd::const_ptr(val) +} + +fn gen_const_long(val: i64) -> lir::Opnd { + Opnd::Imm(val) +} + +fn gen_const_uint16(val: u16) -> lir::Opnd { + Opnd::UImm(val as u64) +} + +fn gen_const_uint32(val: u32) -> lir::Opnd { + Opnd::UImm(val as u64) +} + /// Compile a basic block argument fn gen_param(asm: &mut Assembler, idx: usize) -> lir::Opnd { - asm.live_reg_opnd(Opnd::Reg(param_reg(idx))) + // Allocate a register or a stack slot + match param_opnd(idx) { + // If it's a register, insert LiveReg instruction to reserve the register + // in the register pool for register allocation. + param @ Opnd::Reg(_) => asm.live_reg_opnd(param), + param => param, + } } /// Compile a jump to a basic block -fn gen_jump(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) -> Option<()> { +fn gen_jump(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { // Set basic block arguments gen_branch_params(jit, asm, branch); // Jump to the basic block let target = jit.get_label(asm, branch.target); asm.jmp(target); - Some(()) } /// Compile a conditional branch to a basic block -fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) -> Option<()> { +fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { // If val is zero, move on to the next instruction. let if_false = asm.new_label("if_false"); asm.test(val, val); @@ -414,12 +1260,10 @@ fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: asm.jmp(if_true); asm.write_label(if_false); - - Some(()) } /// Compile a conditional branch to a basic block -fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) -> Option<()> { +fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { // If val is not zero, move on to the next instruction. let if_true = asm.new_label("if_true"); asm.test(val, val); @@ -432,74 +1276,286 @@ fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: asm.jmp(if_false); asm.write_label(if_true); +} + +/// Compile a dynamic dispatch with block +fn gen_send( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_send, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a dynamic dispatch with `...` +fn gen_send_forward( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); - Some(()) + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_sendforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_sendforward, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) } /// Compile a dynamic dispatch without block fn gen_send_without_block( jit: &mut JITState, asm: &mut Assembler, - call_info: &CallInfo, cd: *const rb_call_data, state: &FrameState, -) -> Option<lir::Opnd> { - // Spill the virtual stack onto the stack. They need to be marked by GC and may be caller-saved registers. - // TODO: Avoid spilling operands that have been spilled before. - for (idx, &insn_id) in state.stack().enumerate() { - // Currently, we don't move the SP register. So it's equal to the base pointer. - let stack_opnd = Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32); - asm.mov(stack_opnd, jit.get_opnd(insn_id)?); - } - - // Save PC and SP - gen_save_pc(asm, state); - gen_save_sp(asm, state); + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); - asm_comment!(asm, "call #{} with dynamic dispatch", call_info.method_name); + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; } - let ret = asm.ccall( - rb_vm_opt_send_without_block as *const u8, - vec![EC, CFP, (cd as usize).into()], - ); - // TODO(max): Add a PatchPoint here that can side-exit the function if the callee messed with - // the frame's locals - - Some(ret) + asm_ccall!( + asm, + rb_vm_opt_send_without_block, + EC, CFP, Opnd::const_ptr(cd) + ) } -/// Compile a direct jump to an ISEQ call without block -fn gen_send_without_block_direct( +/// Compile a direct call to an ISEQ method. +/// If `block_handler` is provided, it's used as the specval for the new frame (for forwarding blocks). +/// Otherwise, `VM_BLOCK_HANDLER_NONE` is used. +fn gen_send_iseq_direct( cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, + cme: *const rb_callable_method_entry_t, iseq: IseqPtr, recv: Opnd, - args: &Vec<InsnId>, -) -> Option<lir::Opnd> { + args: Vec<Opnd>, + state: &FrameState, + block_handler: Option<Opnd>, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::iseq_optimized_send_count); + + let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); + let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); + gen_stack_overflow_check(jit, asm, state, stack_growth); + + // Save cfp->pc and cfp->sp for the caller frame + // Can't use gen_prepare_non_leaf_call because we need special SP math. + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, state.stack().len() - args.len() - 1); // -1 for receiver + + gen_spill_locals(jit, asm, state); + gen_spill_stack(jit, asm, state); + + let (frame_type, specval) = if VM_METHOD_TYPE_BMETHOD == unsafe { get_cme_def_type(cme) } { + // Extract EP from the Proc instance + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let bmethod_frame_type = VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA; + // Tag the captured EP like VM_GUARDED_PREV_EP() in vm_call_iseq_bmethod() + let bmethod_specval = (capture.ep.addr() | 1).into(); + (bmethod_frame_type, bmethod_specval) + } else { + let specval = block_handler.unwrap_or_else(|| VM_BLOCK_HANDLER_NONE.into()); + (VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, specval) + }; + // Set up the new frame - gen_push_frame(asm, recv); + // TODO: Lazily materialize caller frames on side exits or when needed + gen_push_frame(asm, args.len(), state, ControlFrame { + recv, + iseq: Some(iseq), + cme, + frame_type, + pc: None, + specval, + }); + + // Write "keyword_bits" to the callee's frame if the callee accepts keywords. + // This is a synthetic local/parameter that the callee reads via checkkeyword to determine + // which optional keyword arguments need their defaults evaluated. + if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + let bits_start = unsafe { (*keyword).bits_start } as usize; + // Currently we only support required keywords, so all bits are 0 (all keywords specified). + // TODO: When supporting optional keywords, calculate actual unspecified_bits here. + let unspecified_bits = VALUE::fixnum_from_usize(0); + let bits_offset = (state.stack().len() - args.len() + bits_start) * SIZEOF_VALUE; + asm_comment!(asm, "write keyword bits to callee frame"); + asm.store(Opnd::mem(64, SP, bits_offset as i32), unspecified_bits.into()); + } + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (state.stack().len() + local_size - args.len() + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); asm_comment!(asm, "switch to new CFP"); let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, new_cfp); - asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); // Set up arguments - let mut c_args: Vec<Opnd> = vec![]; - for &arg in args.iter() { - c_args.push(jit.get_opnd(arg)?); + let mut c_args = vec![recv]; + c_args.extend(&args); + + let params = unsafe { iseq.params() }; + let num_optionals_passed = if params.flags.has_opt() != 0 { + // See vm_call_iseq_setup_normal_opt_start in vm_inshelper.c + let lead_num = params.lead_num as u32; + let opt_num = params.opt_num as u32; + let keyword = params.keyword; + let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } } as u32; + let req_num = lead_num + kw_req_num; + assert!(args.len() as u32 <= req_num + opt_num); + let num_optionals_passed = args.len() as u32 - req_num; + num_optionals_passed + } else { + 0 + }; + + // Fill non-parameter locals with nil (they may be read by eval before being written) + let num_params = params.size.to_usize(); + if local_size > num_params { + asm_comment!(asm, "initialize non-parameter locals to nil"); + for local_idx in num_params..local_size { + let offset = local_size_and_idx_to_bp_offset(local_size, local_idx); + asm.store(Opnd::mem(64, SP, -offset * SIZEOF_VALUE_I32), Qnil.into()); + } } // Make a method call. The target address will be rewritten once compiled. - let branch = Branch::new(); + let iseq_call = IseqCall::new(iseq, num_optionals_passed); let dummy_ptr = cb.get_write_ptr().raw_ptr(cb); - jit.branch_iseqs.push((branch.clone(), iseq)); - // TODO(max): Add a PatchPoint here that can side-exit the function if the callee messed with - // the frame's locals - Some(asm.ccall_with_branch(dummy_ptr, c_args, &branch)) + jit.iseq_calls.push(iseq_call.clone()); + let ret = asm.ccall_with_iseq_call(dummy_ptr, c_args, &iseq_call); + + // If a callee side-exits, i.e. returns Qundef, propagate the return value to the caller. + // The caller will side-exit the callee into the interpreter. + // TODO: Let side exit code pop all JIT frames to optimize away this cmp + je. + asm_comment!(asm, "side-exit if callee side-exits"); + asm.cmp(ret, Qundef.into()); + // Restore the C stack pointer on exit + asm.je(ZJITState::get_exit_trampoline().into()); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + ret +} + +/// Compile for invokeblock +fn gen_invokeblock( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeblock"); + unsafe extern "C" { + fn rb_vm_invokeblock(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokeblock, + EC, CFP, Opnd::const_ptr(cd) + ) +} + +fn gen_invokeproc( + jit: &mut JITState, + asm: &mut Assembler, + recv: Opnd, + args: Vec<Opnd>, + kw_splat: bool, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeproc"); + + let argv_ptr = gen_push_opnds(asm, &args); + let kw_splat_opnd = Opnd::Imm(i64::from(kw_splat)); + let result = asm_ccall!( + asm, + rb_optimized_call, + recv, + EC, + args.len().into(), + argv_ptr, + kw_splat_opnd, + VM_BLOCK_HANDLER_NONE.into() + ); + gen_pop_opnds(asm, &args); + + result +} + +/// Compile a dynamic dispatch for `super` +fn gen_invokesuper( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call super with dynamic dispatch"); + unsafe extern "C" { + fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokesuper, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a string resurrection +fn gen_string_copy(asm: &mut Assembler, recv: Opnd, chilled: bool, state: &FrameState) -> Opnd { + // TODO: split rb_ec_str_resurrect into separate functions + gen_prepare_leaf_call_with_gc(asm, state); + let chilled = if chilled { Opnd::Imm(1) } else { Opnd::Imm(0) }; + asm_ccall!(asm, rb_ec_str_resurrect, EC, recv, chilled) } /// Compile an array duplication instruction @@ -508,86 +1564,310 @@ fn gen_array_dup( val: lir::Opnd, state: &FrameState, ) -> lir::Opnd { - asm_comment!(asm, "call rb_ary_resurrect"); + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_ary_resurrect, val) +} + +/// Compile a new array instruction +fn gen_new_array( + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + let num: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + if elements.is_empty() { + asm_ccall!(asm, rb_ec_ary_new_from_values, EC, 0i64.into(), Opnd::UImm(0)) + } else { + let argv = gen_push_opnds(asm, &elements); + let new_array = asm_ccall!(asm, rb_ec_ary_new_from_values, EC, num.into(), argv); + gen_pop_opnds(asm, &elements); + new_array + } +} + +/// Compile array access (`array[index]`) +fn gen_array_aref( + asm: &mut Assembler, + array: Opnd, + index: Opnd, +) -> lir::Opnd { + let unboxed_idx = asm.load(index); + let array = asm.load(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.load(Opnd::mem(VALUE_BITS, elem_ptr, 0)) +} + +fn gen_array_aset( + asm: &mut Assembler, + array: Opnd, + index: Opnd, + val: Opnd, +) { + let unboxed_idx = asm.load(index); + let array = asm.load(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.store(Opnd::mem(VALUE_BITS, elem_ptr, 0), val); +} + +fn gen_array_pop(asm: &mut Assembler, array: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_pop, array) +} + +fn gen_array_length(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let array = asm.load(array); + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + let embedded_len = asm.and(flags, (RARRAY_EMBED_LEN_MASK as u64).into()); + let embedded_len = asm.rshift(embedded_len, (RARRAY_EMBED_LEN_SHIFT as u64).into()); + // cmov between the embedded length and heap length depending on the embed flag + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_len = Opnd::mem(c_long::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_LEN); + asm.csel_nz(embedded_len, heap_len) +} + +fn gen_array_ptr(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_ptr = Opnd::mem(usize::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_PTR); + let embedded_ptr = asm.lea(Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RARRAY_AS_ARY)); + asm.csel_nz(embedded_ptr, heap_ptr) +} + +/// Compile opt_newarray_hash - create a hash from array elements +fn gen_opt_newarray_hash( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + // `Array#hash` will hash the elements of the array. + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); - // Save PC - gen_save_pc(asm, state); + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_hash(ec: EcPtr, array_len: u32, elts: *const VALUE) -> VALUE; + } asm.ccall( - rb_ary_resurrect as *const u8, - vec![val], + rb_vm_opt_newarray_hash as *const u8, + vec![EC, (array_len as u32).into(), elements_ptr], ) } -/// Compile a new array instruction -fn gen_new_array( +fn gen_array_include( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the target. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len() - 1; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_include_p(ec: EcPtr, num: c_long, elts: *const VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_include_p, + EC, array_len.into(), elements_ptr, target + ) +} + +fn gen_array_pack_buffer( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + fmt: Opnd, + buffer: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the fmt, followed by the buffer. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len() - 2; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_pack_buffer(ec: EcPtr, num: c_long, elts: *const VALUE, fmt: VALUE, buffer: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_pack_buffer, + EC, array_len.into(), elements_ptr, fmt, buffer + ) +} + +fn gen_dup_array_include( + jit: &JITState, + asm: &mut Assembler, + ary: VALUE, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + unsafe extern "C" { + fn rb_vm_opt_duparray_include_p(ec: EcPtr, ary: VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_duparray_include_p, + EC, ary.into(), target + ) +} + +fn gen_is_a(asm: &mut Assembler, obj: Opnd, class: Opnd) -> lir::Opnd { + asm_ccall!(asm, rb_obj_is_kind_of, obj, class) +} + +/// Compile a new hash instruction +fn gen_new_hash( jit: &mut JITState, asm: &mut Assembler, - elements: &Vec<InsnId>, + elements: Vec<Opnd>, state: &FrameState, ) -> lir::Opnd { - asm_comment!(asm, "call rb_ary_new"); + gen_prepare_non_leaf_call(jit, asm, state); - // Save PC - gen_save_pc(asm, state); + let cap: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + let new_hash = asm_ccall!(asm, rb_hash_new_with_size, lir::Opnd::Imm(cap)); - let length: ::std::os::raw::c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + if !elements.is_empty() { + let argv = gen_push_opnds(asm, &elements); + asm_ccall!(asm, rb_hash_bulk_insert, elements.len().into(), argv, new_hash); - let new_array = asm.ccall( - rb_ary_new_capa as *const u8, - vec![lir::Opnd::Imm(length)], - ); + gen_pop_opnds(asm, &elements); + } - for i in 0..elements.len() { - let insn_id = elements.get(i as usize).expect("Element should exist at index"); - let val = jit.get_opnd(*insn_id).unwrap(); - asm.ccall( - rb_ary_push as *const u8, - vec![new_array, val] - ); + new_hash +} + +/// Compile a new range instruction +fn gen_new_range( + jit: &JITState, + asm: &mut Assembler, + low: lir::Opnd, + high: lir::Opnd, + flag: RangeType, + state: &FrameState, +) -> lir::Opnd { + // Sometimes calls `low.<=>(high)` + gen_prepare_non_leaf_call(jit, asm, state); + + // Call rb_range_new(low, high, flag) + asm_ccall!(asm, rb_range_new, low, high, (flag as i32).into()) +} + +fn gen_new_range_fixnum( + asm: &mut Assembler, + low: lir::Opnd, + high: lir::Opnd, + flag: RangeType, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_range_new, low, high, (flag as i64).into()) +} + +fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Allocating an object from an unknown class is non-leaf; see doc for `ObjectAlloc`. + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_obj_alloc, val) +} + +fn gen_object_alloc_class(asm: &mut Assembler, class: VALUE, state: &FrameState) -> lir::Opnd { + // Allocating an object for a known class with default allocator is leaf; see doc for + // `ObjectAllocClass`. + gen_prepare_leaf_call_with_gc(asm, state); + if unsafe { rb_zjit_class_has_default_allocator(class) } { + // TODO(max): inline code to allocate an instance + asm_ccall!(asm, rb_class_allocate_instance, class.into()) + } else { + assert!(class_has_leaf_allocator(class), "class passed to ObjectAllocClass must have a leaf allocator"); + let alloc_func = unsafe { rb_zjit_class_get_alloc_func(class) }; + assert!(alloc_func.is_some(), "class {} passed to ObjectAllocClass must have an allocator", get_class_name(class)); + asm_comment!(asm, "call allocator for class {}", get_class_name(class)); + asm.count_call_to(&format!("{}::allocator", get_class_name(class))); + asm.ccall(alloc_func.unwrap() as *const u8, vec![class.into()]) } +} - new_array +/// Compile a frame setup. If jit_entry_idx is Some, remember the address of it as a JIT entry. +fn gen_entry_point(jit: &mut JITState, asm: &mut Assembler, jit_entry_idx: Option<usize>) { + if let Some(jit_entry_idx) = jit_entry_idx { + let jit_entry = JITEntry::new(jit_entry_idx); + jit.jit_entries.push(jit_entry.clone()); + asm.pos_marker(move |code_ptr, _| { + jit_entry.borrow_mut().start_addr.set(Some(code_ptr)); + }); + } + asm.frame_setup(&[]); } /// Compile code that exits from JIT code with a return value -fn gen_return(asm: &mut Assembler, val: lir::Opnd) -> Option<()> { +fn gen_return(asm: &mut Assembler, val: lir::Opnd) { // Pop the current frame (ec->cfp++) // Note: the return PC is already in the previous CFP asm_comment!(asm, "pop stack frame"); let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, incr_cfp); - asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); - asm.frame_teardown(); + // Order here is important. Because we're about to tear down the frame, + // we need to load the return value, which might be part of the frame. + asm.load_into(C_RET_OPND, val); // Return from the function - asm.cret(val); - Some(()) + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(C_RET_OPND); } /// Compile Fixnum + Fixnum -fn gen_fixnum_add(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Add left + right and test for overflow let left_untag = asm.sub(left, Opnd::Imm(1)); let out_val = asm.add(left_untag, right); - asm.jo(Target::SideExit(state.clone())); + asm.jo(side_exit(jit, state, FixnumAddOverflow)); - Some(out_val) + out_val } /// Compile Fixnum - Fixnum -fn gen_fixnum_sub(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Subtract left - right and test for overflow let val_untag = asm.sub(left, right); - asm.jo(Target::SideExit(state.clone())); - let out_val = asm.add(val_untag, Opnd::Imm(1)); - - Some(out_val) + asm.jo(side_exit(jit, state, FixnumSubOverflow)); + asm.add(val_untag, Opnd::Imm(1)) } /// Compile Fixnum * Fixnum -fn gen_fixnum_mult(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> Option<lir::Opnd> { +fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Do some bitwise gymnastics to handle tag bits // x * y is translated to (x >> 1) * (y - 1) + 1 let left_untag = asm.rshift(left, Opnd::UImm(1)); @@ -595,180 +1875,983 @@ fn gen_fixnum_mult(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state let out_val = asm.mul(left_untag, right_untag); // Test for overflow - asm.jo_mul(Target::SideExit(state.clone())); - let out_val = asm.add(out_val, Opnd::UImm(1)); + asm.jo_mul(side_exit(jit, state, FixnumMultOverflow)); + asm.add(out_val, Opnd::UImm(1)) +} + +/// Compile Fixnum / Fixnum +fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); - Some(out_val) + // Side exit if rhs is 0 + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(side_exit(jit, state, FixnumDivByZero)); + asm_ccall!(asm, rb_jit_fix_div_fix, left, right) } /// Compile Fixnum == Fixnum -fn gen_fixnum_eq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_eq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_e(Qtrue.into(), Qfalse.into())) + asm.csel_e(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum != Fixnum -fn gen_fixnum_neq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_neq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_ne(Qtrue.into(), Qfalse.into())) + asm.csel_ne(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum < Fixnum -fn gen_fixnum_lt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_lt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_l(Qtrue.into(), Qfalse.into())) + asm.csel_l(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum <= Fixnum -fn gen_fixnum_le(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_le(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_le(Qtrue.into(), Qfalse.into())) + asm.csel_le(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum > Fixnum -fn gen_fixnum_gt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_gt(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_g(Qtrue.into(), Qfalse.into())) + asm.csel_g(Qtrue.into(), Qfalse.into()) } /// Compile Fixnum >= Fixnum -fn gen_fixnum_ge(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> Option<lir::Opnd> { +fn gen_fixnum_ge(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ge(Qtrue.into(), Qfalse.into()) +} + +/// Compile Fixnum & Fixnum +fn gen_fixnum_and(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.and(left, right) +} + +/// Compile Fixnum | Fixnum +fn gen_fixnum_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.or(left, right) +} + +/// Compile Fixnum ^ Fixnum +fn gen_fixnum_xor(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + // XOR and then re-tag the resulting fixnum + let out_val = asm.xor(left, right); + asm.add(out_val, Opnd::UImm(1)) +} + +/// Compile Fixnum << Fixnum +fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let in_val = asm.sub(left, Opnd::UImm(1)); // Drop tag bit + let out_val = asm.lshift(in_val, shift_amount.into()); + let unshifted = asm.rshift(out_val, shift_amount.into()); + asm.cmp(in_val, unshifted); + asm.jne(side_exit(jit, state, FixnumLShiftOverflow)); + // Re-tag the output value + let out_val = asm.add(out_val, 1.into()); + out_val +} + +/// Compile Fixnum >> Fixnum +fn gen_fixnum_rshift(asm: &mut Assembler, left: lir::Opnd, shift_amount: u64) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let result = asm.rshift(left, shift_amount.into()); + // Re-tag the output value + asm.or(result, 1.into()) +} + +fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Check for left % 0, which raises ZeroDivisionError + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(side_exit(jit, state, FixnumModByZero)); + asm_ccall!(asm, rb_fix_mod_fix, left, right) +} + +fn gen_fixnum_aref(asm: &mut Assembler, recv: lir::Opnd, index: lir::Opnd) -> lir::Opnd { + asm_ccall!(asm, rb_fix_aref, recv, index) +} + +// Compile val == nil +fn gen_isnil(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + asm.cmp(val, Qnil.into()); + // TODO: Implement and use setcc + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_is_method_cfunc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, cd: *const rb_call_data, cfunc: *const u8) -> lir::Opnd { + unsafe extern "C" { + fn rb_vm_method_cfunc_is(iseq: IseqPtr, cd: *const rb_call_data, recv: VALUE, cfunc: *const u8) -> VALUE; + } + asm_ccall!(asm, rb_vm_method_cfunc_is, VALUE::from(jit.iseq).into(), Opnd::const_ptr(cd), val, Opnd::const_ptr(cfunc)) +} + +fn gen_is_bit_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); - Some(asm.csel_ge(Qtrue.into(), Qfalse.into())) + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_is_bit_not_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ne(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_box_bool(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + asm.test(val, val); + asm.csel_nz(Opnd::Value(Qtrue), Opnd::Value(Qfalse)) +} + +fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Load the value, then test for overflow and tag it + let val = asm.load(val); + let shifted = asm.lshift(val, Opnd::UImm(1)); + asm.jo(side_exit(jit, state, BoxFixnumOverflow)); + asm.or(shifted, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)) +} + +fn gen_anytostring(asm: &mut Assembler, val: lir::Opnd, str: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + asm_ccall!(asm, rb_obj_as_string_result, str, val) } /// Evaluate if a value is truthy /// Produces a CBool type (0 or 1) /// In Ruby, only nil and false are falsy /// Everything else evaluates to true -fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> Option<lir::Opnd> { +fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { // Test if any bit (outside of the Qnil bit) is on // See RB_TEST(), include/ruby/internal/special_consts.h asm.test(val, Opnd::Imm(!Qnil.as_i64())); - Some(asm.csel_e(0.into(), 1.into())) + asm.csel_e(0.into(), 1.into()) } /// Compile a type check with a side exit -fn gen_guard_type(asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> Option<lir::Opnd> { - if guard_type.is_subtype(Fixnum) { - // Check if opnd is Fixnum +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> lir::Opnd { + gen_incr_counter(asm, Counter::guard_type_count); + if guard_type.is_subtype(types::Fixnum) { asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); - asm.jz(Target::SideExit(state.clone())); + asm.jz(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::Flonum) { + // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG + let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); + asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::StaticSymbol) { + // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG + // Use 8-bit comparison like YJIT does. GuardType should not be used + // for a known VALUE, which with_num_bits() does not support. + asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::NilClass) { + asm.cmp(val, Qnil.into()); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::TrueClass) { + asm.cmp(val, Qtrue.into()); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_subtype(types::FalseClass) { + asm.cmp(val, Qfalse.into()); + asm.jne(side_exit(jit, state, GuardType(guard_type))); + } else if guard_type.is_immediate() { + // All immediate types' guard should have been handled above + panic!("unexpected immediate guard type: {guard_type}"); + } else if let Some(expected_class) = guard_type.runtime_exact_ruby_class() { + asm_comment!(asm, "guard exact class for non-immediate types"); + + // If val isn't in a register, load it to use it as the base of Opnd::mem later. + // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) + let val = match val { + Opnd::Reg(_) | Opnd::VReg { .. } => val, + _ => asm.load(val), + }; + + // Check if it's a special constant + let side_exit = side_exit(jit, state, GuardType(guard_type)); + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(side_exit.clone()); + + // Check if it's false + asm.cmp(val, Qfalse.into()); + asm.je(side_exit.clone()); + + // Load the class from the object's klass field + let klass = asm.load(Opnd::mem(64, val, RUBY_OFFSET_RBASIC_KLASS)); + + asm.cmp(klass, Opnd::Value(expected_class)); + asm.jne(side_exit); + } else if guard_type.is_subtype(types::String) { + let side = side_exit(jit, state, GuardType(guard_type)); + + // Check special constant + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(side.clone()); + + // Check false + asm.cmp(val, Qfalse.into()); + asm.je(side.clone()); + + let val = match val { + Opnd::Reg(_) | Opnd::VReg { .. } => val, + _ => asm.load(val), + }; + + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let tag = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(tag, Opnd::UImm(RUBY_T_STRING as u64)); + asm.jne(side); + } else if guard_type.bit_equal(types::HeapBasicObject) { + let side_exit = side_exit(jit, state, GuardType(guard_type)); + asm.cmp(val, Opnd::Value(Qfalse)); + asm.je(side_exit.clone()); + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(side_exit); + } else { + unimplemented!("unsupported type: {guard_type}"); + } + val +} + +fn gen_guard_type_not(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> lir::Opnd { + if guard_type.is_subtype(types::String) { + // We only exit if val *is* a String. Otherwise we fall through. + let cont = asm.new_label("guard_type_not_string_cont"); + let side = side_exit(jit, state, GuardTypeNot(guard_type)); + + // Continue if special constant (not string) + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(cont.clone()); + + // Continue if false (not string) + asm.cmp(val, Qfalse.into()); + asm.je(cont.clone()); + + let val = match val { + Opnd::Reg(_) | Opnd::VReg { .. } => val, + _ => asm.load(val), + }; + + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let tag = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(tag, Opnd::UImm(RUBY_T_STRING as u64)); + asm.je(side); + + // Otherwise (non-string heap object), continue. + asm.write_label(cont); } else { unimplemented!("unsupported type: {guard_type}"); } - Some(val) + val } /// Compile an identity check with a side exit -fn gen_guard_bit_equals(asm: &mut Assembler, val: lir::Opnd, expected: VALUE, state: &FrameState) -> Option<lir::Opnd> { - asm.cmp(val, Opnd::UImm(expected.into())); - asm.jnz(Target::SideExit(state.clone())); - Some(val) +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let expected_opnd: Opnd = match expected { + crate::hir::Const::Value(v) => { Opnd::Value(v) } + crate::hir::Const::CInt64(v) => { v.into() } + crate::hir::Const::CShape(v) => { Opnd::UImm(v.0 as u64) } + _ => panic!("gen_guard_bit_equals: unexpected hir::Const {expected:?}"), + }; + asm.cmp(val, expected_opnd); + asm.jnz(side_exit(jit, state, reason)); + val +} + +/// Generate code that records unoptimized C functions if --zjit-stats is enabled +fn gen_incr_counter_ptr(asm: &mut Assembler, counter_ptr: *mut u64) { + if get_option!(stats) { + asm.incr_counter(Opnd::const_ptr(counter_ptr as *const u8), Opnd::UImm(1)); + } +} + +/// Generate code that increments a counter if --zjit-stats +fn gen_incr_counter(asm: &mut Assembler, counter: Counter) { + if get_option!(stats) { + let ptr = counter_ptr(counter); + gen_incr_counter_ptr(asm, ptr); + } } -/// Save the incremented PC on the CFP. -/// This is necessary when callees can raise or allocate. -fn gen_save_pc(asm: &mut Assembler, state: &FrameState) { +/// Increment a counter for each DynamicSendReason. If the variant has +/// a counter prefix to break down the details, increment that as well. +fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReason) { + gen_incr_counter(asm, send_fallback_counter(reason)); + + use SendFallbackReason::*; + match reason { + Uncategorized(opcode) => { + gen_incr_counter_ptr(asm, send_fallback_counter_ptr_for_opcode(opcode)); + } + SendWithoutBlockNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_method_type(method_type)); + } + SendWithoutBlockNotOptimizedMethodTypeOptimized(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_optimized_method_type(method_type)); + } + SendNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_method_type(method_type)); + } + SuperNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_super_method_type(method_type)); + } + _ => {} + } +} + +/// Save only the PC to CFP. Use this when you need to call gen_save_sp() +/// immediately after with a custom stack size (e.g., gen_ccall_with_frame +/// adjusts SP to exclude receiver and arguments). +fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { let opcode: usize = state.get_opcode().try_into().unwrap(); let next_pc: *const VALUE = unsafe { state.pc.offset(insn_len(opcode) as isize) }; + gen_incr_counter(asm, Counter::vm_write_pc_count); asm_comment!(asm, "save PC to CFP"); - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(next_pc as *const u8)); + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(next_pc)); +} + +/// Save the current PC on the CFP as a preparation for calling a C function +/// that may allocate objects and trigger GC. Use gen_prepare_non_leaf_call() +/// if it may raise exceptions or call arbitrary methods. +/// +/// Unlike YJIT, we don't need to save the stack slots to protect them from GC +/// because the backend spills all live registers onto the C stack on CCall. +/// However, to avoid marking uninitialized stack slots, this also updates SP, +/// which may have cfp->sp for a past frame or a past non-leaf call. +fn gen_prepare_call_with_gc(asm: &mut Assembler, state: &FrameState, leaf: bool) { + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, state.stack_size()); + if leaf { + asm.expect_leaf_ccall(state.stack_size()); + } +} + +fn gen_prepare_leaf_call_with_gc(asm: &mut Assembler, state: &FrameState) { + // In gen_prepare_call_with_gc(), we update cfp->sp for leaf calls too. + // + // Here, cfp->sp may be pointing to either of the following: + // 1. cfp->sp for a past frame, which gen_push_frame() skips to initialize + // 2. cfp->sp set by gen_prepare_non_leaf_call() for the current frame + // + // When (1), to avoid marking dead objects, we need to set cfp->sp for the current frame. + // When (2), setting cfp->sp at gen_push_frame() and not updating cfp->sp here could lead to + // keeping objects longer than it should, so we set cfp->sp at every call of this function. + // + // We use state.without_stack() to pass stack_size=0 to gen_save_sp() because we don't write + // VM stack slots on leaf calls, which leaves those stack slots uninitialized. ZJIT keeps + // live objects on the C stack, so they are protected from GC properly. + gen_prepare_call_with_gc(asm, &state.without_stack(), true); } /// Save the current SP on the CFP -fn gen_save_sp(asm: &mut Assembler, state: &FrameState) { +fn gen_save_sp(asm: &mut Assembler, stack_size: usize) { // Update cfp->sp which will be read by the interpreter. We also have the SP register in JIT // code, and ZJIT's codegen currently assumes the SP register doesn't move, e.g. gen_param(). // So we don't update the SP register here. We could update the SP register to avoid using // an extra register for asm.lea(), but you'll need to manage the SP offset like YJIT does. - asm_comment!(asm, "save SP to CFP: {}", state.stack_size()); - let sp_addr = asm.lea(Opnd::mem(64, SP, state.stack_size() as i32 * SIZEOF_VALUE_I32)); + gen_incr_counter(asm, Counter::vm_write_sp_count); + asm_comment!(asm, "save SP to CFP: {}", stack_size); + let sp_addr = asm.lea(Opnd::mem(64, SP, stack_size as i32 * SIZEOF_VALUE_I32)); let cfp_sp = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); asm.mov(cfp_sp, sp_addr); } +/// Spill locals onto the stack. +fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // TODO: Avoid spilling locals that have been spilled before and not changed. + gen_incr_counter(asm, Counter::vm_write_locals_count); + asm_comment!(asm, "spill locals"); + for (idx, &insn_id) in state.locals().enumerate() { + asm.mov(Opnd::mem(64, SP, (-local_idx_to_ep_offset(jit.iseq, idx) - 1) * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + } +} + +/// Spill the virtual stack onto the stack. +fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // This function does not call gen_save_sp() at the moment because + // gen_send_without_block_direct() spills stack slots above SP for arguments. + gen_incr_counter(asm, Counter::vm_write_stack_count); + asm_comment!(asm, "spill stack"); + for (idx, &insn_id) in state.stack().enumerate() { + asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + } +} + +/// Prepare for calling a C function that may call an arbitrary method. +/// Use gen_prepare_leaf_call_with_gc() if the method is leaf but allocates objects. +fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { + // TODO: Lazily materialize caller frames when needed + // Save PC for backtraces and allocation tracing + // and SP to avoid marking uninitialized stack slots + gen_prepare_call_with_gc(asm, state, false); + + // Spill the virtual stack in case it raises an exception + // and the interpreter uses the stack for handling the exception + gen_spill_stack(jit, asm, state); + + // Spill locals in case the method looks at caller Bindings + gen_spill_locals(jit, asm, state); +} + +/// Frame metadata written by gen_push_frame() +struct ControlFrame { + recv: Opnd, + iseq: Option<IseqPtr>, + cme: *const rb_callable_method_entry_t, + frame_type: u32, + /// The [`VM_ENV_DATA_INDEX_SPECVAL`] slot of the frame. + /// For the type of frames we push, block handler or the parent EP. + specval: lir::Opnd, + pc: Option<*const VALUE>, +} + /// Compile an interpreter frame -fn gen_push_frame(asm: &mut Assembler, recv: Opnd) { - // Write to a callee CFP +fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: ControlFrame) { + // Locals are written by the callee frame on side-exits or non-leaf calls + + // See vm_push_frame() for details + asm_comment!(asm, "push cme, specval, frame type"); + // ep[-2]: cref of cme + let local_size = if let Some(iseq) = frame.iseq { + (unsafe { get_iseq_body_local_table_size(iseq) }) as i32 + } else { + 0 + }; + let ep_offset = state.stack().len() as i32 + local_size - argc as i32 + VM_ENV_DATA_SIZE as i32 - 1; + // ep[-2]: CME + asm.store(Opnd::mem(64, SP, (ep_offset - 2) * SIZEOF_VALUE_I32), VALUE::from(frame.cme).into()); + // ep[-1]: specval + asm.store(Opnd::mem(64, SP, (ep_offset - 1) * SIZEOF_VALUE_I32), frame.specval); + // ep[0]: ENV_FLAGS + asm.store(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32), frame.frame_type.into()); + + // Write to the callee CFP fn cfp_opnd(offset: i32) -> Opnd { Opnd::mem(64, CFP, offset - (RUBY_SIZEOF_CONTROL_FRAME as i32)) } asm_comment!(asm, "push callee control frame"); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), recv); - // TODO: Write more fields as needed + + if let Some(iseq) = frame.iseq { + // cfp_opnd(RUBY_OFFSET_CFP_PC): written by the callee frame on side-exits, non-leaf calls, or calls with GC + // cfp_opnd(RUBY_OFFSET_CFP_SP): written by the callee frame on side-exits, non-leaf calls, or calls with GC + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), VALUE::from(iseq).into()); + } else { + // C frames don't have a PC and ISEQ in normal operation. + // When runtime checks are enabled we poison the PC so accidental reads stand out. + if let Some(pc) = frame.pc { + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + } + let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), 0.into()); + } + + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); + let ep = asm.lea(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32)); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); } -/// Return a register we use for the basic block argument at a given index -fn param_reg(idx: usize) -> Reg { - // To simplify the implementation, allocate a fixed register for each basic block argument for now. +/// Stack overflow check: fails if CFP<=SP at any point in the callee. +fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &FrameState, stack_growth: usize) { + asm_comment!(asm, "stack overflow check"); + // vm_push_frame() checks it against a decremented cfp, and CHECK_VM_STACK_OVERFLOW0 + // adds to the margin another control frame with `&bounds[1]`. + const { assert!(RUBY_SIZEOF_CONTROL_FRAME % SIZEOF_VALUE == 0, "sizeof(rb_control_frame_t) is a multiple of sizeof(VALUE)"); } + let cfp_growth = 2 * (RUBY_SIZEOF_CONTROL_FRAME / SIZEOF_VALUE); + let peak_offset = (cfp_growth + stack_growth) * SIZEOF_VALUE; + let stack_limit = asm.lea(Opnd::mem(64, SP, peak_offset as i32)); + asm.cmp(CFP, stack_limit); + asm.jbe(side_exit(jit, state, StackOverflow)); +} + +/// Return an operand we use for the basic block argument at a given index +fn param_opnd(idx: usize) -> Opnd { + // To simplify the implementation, allocate a fixed register or a stack slot for each basic block argument for now. + // Note that this is implemented here as opposed to automatically inside LIR machineries. // TODO: Allow allocating arbitrary registers for basic block arguments - ALLOC_REGS[idx] + if idx < ALLOC_REGS.len() { + Opnd::Reg(ALLOC_REGS[idx]) + } else { + // With FrameSetup, the address that NATIVE_BASE_PTR points to stores an old value in the register. + // To avoid clobbering it, we need to start from the next slot, hence `+ 1` for the index. + Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 1) as i32 * -SIZEOF_VALUE_I32) + } } /// Inverse of ep_offset_to_local_idx(). See ep_offset_to_local_idx() for details. -fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { - let local_table_size: i32 = unsafe { get_iseq_body_local_table_size(iseq) } - .try_into() - .unwrap(); - local_table_size - local_idx as i32 - 1 + VM_ENV_DATA_SIZE as i32 +pub fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { + let local_size = unsafe { get_iseq_body_local_table_size(iseq) }; + local_size_and_idx_to_ep_offset(local_size.to_usize(), local_idx) +} + +/// Convert the number of locals and a local index to an offset from the EP +pub fn local_size_and_idx_to_ep_offset(local_size: usize, local_idx: usize) -> i32 { + local_size as i32 - local_idx as i32 - 1 + VM_ENV_DATA_SIZE as i32 +} + +/// Convert the number of locals and a local index to an offset from the BP. +/// We don't move the SP register after entry, so we often use SP as BP. +pub fn local_size_and_idx_to_bp_offset(local_size: usize, local_idx: usize) -> i32 { + local_size_and_idx_to_ep_offset(local_size, local_idx) + 1 } /// Convert ISEQ into High-level IR -fn compile_iseq(iseq: IseqPtr) -> Option<Function> { +fn compile_iseq(iseq: IseqPtr) -> Result<Function, CompileError> { + // Convert ZJIT instructions back to bare instructions + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + + // Reject ISEQs with very large temp stacks. + // We cannot encode too large offsets to access locals in arm64. + let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) }; + if stack_max >= i8::MAX as u32 { + debug!("ISEQ stack too large: {stack_max}"); + return Err(CompileError::IseqStackTooLarge); + } + let mut function = match iseq_to_hir(iseq) { Ok(function) => function, Err(err) => { - debug!("ZJIT: iseq_to_hir: {err:?}"); - return None; + debug!("ZJIT: iseq_to_hir: {err:?}: {}", iseq_get_location(iseq, 0)); + return Err(CompileError::ParseError(err)); } }; - function.optimize(); - Some(function) + if !get_option!(disable_hir_opt) { + function.optimize(); + } + function.dump_hir(); + Ok(function) +} + +/// Build a Target::SideExit +fn side_exit(jit: &JITState, state: &FrameState, reason: SideExitReason) -> Target { + let exit = build_side_exit(jit, state); + Target::SideExit { exit, reason } +} + +/// Build a side-exit context +fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { + let mut stack = Vec::new(); + for &insn_id in state.stack() { + stack.push(jit.get_opnd(insn_id)); + } + + let mut locals = Vec::new(); + for &insn_id in state.locals() { + locals.push(jit.get_opnd(insn_id)); + } + + SideExit{ + pc: Opnd::const_ptr(state.pc), + stack, + locals, + } +} + +/// Returne the maximum number of arguments for a block in a given function +fn max_num_params(function: &Function) -> usize { + let reverse_post_order = function.rpo(); + reverse_post_order.iter().map(|&block_id| { + let block = function.block(block_id); + block.params().len() + }).max().unwrap_or(0) +} + +#[cfg(target_arch = "x86_64")] +macro_rules! c_callable { + ($(#[$outer:meta])* + fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $(#[$outer])* + extern "sysv64" fn $f $args $(-> $ret)? $body + }; +} +#[cfg(target_arch = "aarch64")] +macro_rules! c_callable { + ($(#[$outer:meta])* + fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $(#[$outer])* + extern "C" fn $f $args $(-> $ret)? $body + }; +} +#[cfg(test)] +pub(crate) use c_callable; + +c_callable! { + /// Generated code calls this function with the SysV calling convention. See [gen_function_stub]. + /// This function is expected to be called repeatedly when ZJIT fails to compile the stub. + /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported + /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. + fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE) -> *const u8 { + with_vm_lock(src_loc!(), || { + // gen_push_frame() doesn't set PC, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); + let entry_insn_idxs = crate::hir::jit_entry_insns(iseq); + let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; + unsafe { rb_set_cfp_pc(cfp, pc) }; + + // Successful JIT-to-JIT calls fill nils to non-parameter locals in generated code. + // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. + fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, compile_error: &CompileError) { + unsafe { + // Set SP which gen_push_frame() doesn't set + rb_set_cfp_sp(cfp, sp); + + // Fill nils to uninitialized (non-argument) locals + let local_size = get_iseq_body_local_table_size(iseq).to_usize(); + let num_params = iseq.params().size.to_usize(); + let base = sp.offset(-local_size_and_idx_to_bp_offset(local_size, num_params) as isize); + slice::from_raw_parts_mut(base, local_size - num_params).fill(Qnil); + } + + // Increment a compile error counter for --zjit-stats + if get_option!(stats) { + incr_counter_by(exit_counter_for_compile_error(compile_error), 1); + } + } + + // If we already know we can't compile the ISEQ, fail early without cb.mark_all_executable(). + // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole + // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. + let cb = ZJITState::get_code_block(); + let payload = get_or_create_iseq_payload(iseq); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + let compile_error = match last_status { + Some(IseqStatus::CantCompile(err)) => Some(err), + _ if cb.has_dropped_bytes() => Some(&CompileError::OutOfMemory), + _ => None, + }; + if let Some(compile_error) = compile_error { + // We'll use this Rc again, so increment the ref count decremented by from_raw. + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + + prepare_for_exit(iseq, cfp, sp, compile_error); + return ZJITState::get_exit_trampoline_with_counter().raw_ptr(cb); + } + + // Otherwise, attempt to compile the ISEQ. We have to mark_all_executable() beyond this point. + let code_ptr = with_time_stat(compile_time_ns, || function_stub_hit_body(cb, &iseq_call)); + if code_ptr.is_ok() { + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.incoming.push(iseq_call); + } + } + let code_ptr = code_ptr.unwrap_or_else(|compile_error| { + // We'll use this Rc again, so increment the ref count decremented by from_raw. + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + + prepare_for_exit(iseq, cfp, sp, &compile_error); + ZJITState::get_exit_trampoline_with_counter() + }); + cb.mark_all_executable(); + code_ptr.raw_ptr(cb) + }) + } +} + +/// Compile an ISEQ for a function stub +fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<CodePtr, CompileError> { + // Compile the stubbed ISEQ + let IseqCodePtrs { jit_entry_ptrs, .. } = gen_iseq(cb, iseq_call.iseq.get(), None).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + + // Update the stub to call the code pointer + let jit_entry_ptr = jit_entry_ptrs[iseq_call.jit_entry_idx.to_usize()]; + let code_addr = jit_entry_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call compiled function: {}", iseq_get_location(iseq, 0)); + asm.ccall(code_addr, vec![]); + }); + + Ok(jit_entry_ptr) +} + +/// Compile a stub for an ISEQ called by SendWithoutBlockDirect +fn gen_function_stub(cb: &mut CodeBlock, iseq_call: IseqCallRef) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + + // Call function_stub_hit using the shared trampoline. See `gen_function_stub_hit_trampoline`. + // Use load_into instead of mov, which is split on arm64, to avoid clobbering ALLOC_REGS. + asm.load_into(scratch_reg, Opnd::const_ptr(Rc::into_raw(iseq_call))); + asm.jmp(ZJITState::get_function_stub_hit_trampoline().into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that is used when a function stub is called. +/// See [gen_function_stub] for how it's used. +pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm_comment!(asm, "function_stub_hit trampoline"); + + // Maintain alignment for x86_64, and set up a frame for arm64 properly + asm.frame_setup(&[]); + + asm_comment!(asm, "preserve argument registers"); + for ® in ALLOC_REGS.iter() { + asm.cpush(Opnd::Reg(reg)); + } + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpush(Opnd::Reg(ALLOC_REGS[0])); // maintain alignment for x86_64 + } + + // Compile the stubbed ISEQ + let jump_addr = asm_ccall!(asm, function_stub_hit, scratch_reg, CFP, SP); + asm.mov(scratch_reg, jump_addr); + + asm_comment!(asm, "restore argument registers"); + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpop_into(Opnd::Reg(ALLOC_REGS[0])); + } + for ® in ALLOC_REGS.iter().rev() { + asm.cpop_into(Opnd::Reg(reg)); + } + + // Discard the current frame since the JIT function will set it up again + asm.frame_teardown(&[]); + + // Jump to scratch_reg so that cpop_into() doesn't clobber it + asm.jmp_opnd(scratch_reg); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that is used when a function exits without restoring PC and the stack +pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + + asm_comment!(asm, "side-exit trampoline"); + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() + asm.cret(Qundef.into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +/// Generate a trampoline that increments exit_compilation_failure and jumps to exit_trampoline. +pub fn gen_exit_trampoline_with_counter(cb: &mut CodeBlock, exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + + asm_comment!(asm, "function stub exit trampoline"); + gen_incr_counter(&mut asm, exit_compile_error); + asm.jmp(Target::CodePtr(exit_trampoline)); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} + +fn gen_push_opnds(asm: &mut Assembler, opnds: &[Opnd]) -> lir::Opnd { + let n = opnds.len(); + let allocation_size = aligned_stack_bytes(n); + + // Bump the stack pointer to reserve the space for opnds + if n != 0 { + asm_comment!(asm, "allocate {} bytes on C stack for {} values", allocation_size, n); + asm.sub_into(NATIVE_STACK_PTR, allocation_size.into()); + } else { + asm_comment!(asm, "no opnds to allocate"); + } + + // Load NATIVE_STACK_PTR to get the address of a returned array + // to allow the backend to move it for its own use. + let argv = asm.load(NATIVE_STACK_PTR); + for (idx, &opnd) in opnds.iter().enumerate() { + asm.mov(Opnd::mem(VALUE_BITS, argv, idx as i32 * SIZEOF_VALUE_I32), opnd); + } + + argv +} + +fn gen_pop_opnds(asm: &mut Assembler, opnds: &[Opnd]) { + if opnds.is_empty() { + asm_comment!(asm, "no opnds to restore"); + return + } + + asm_comment!(asm, "restore C stack pointer"); + let allocation_size = aligned_stack_bytes(opnds.len()); + asm.add_into(NATIVE_STACK_PTR, allocation_size.into()); +} + +fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, opt: usize, values: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let first_opnd_ptr = gen_push_opnds(asm, &values); + + let tmp_ary = asm_ccall!(asm, rb_ary_tmp_new_from_values, Opnd::Imm(0), values.len().into(), first_opnd_ptr); + let result = asm_ccall!(asm, rb_reg_new_ary, tmp_ary, opt.into()); + asm_ccall!(asm, rb_ary_clear, tmp_ary); + + gen_pop_opnds(asm, &values); + + result +} + +fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd>, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let first_string_ptr = gen_push_opnds(asm, &strings); + let result = asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr); + gen_pop_opnds(asm, &strings); + + result +} + +// Generate RSTRING_PTR +fn get_string_ptr(asm: &mut Assembler, string: Opnd) -> Opnd { + asm_comment!(asm, "get string pointer for embedded or heap"); + let string = asm.load(string); + let flags = Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RSTRING_NOEMBED as u64).into()); + let heap_ptr = asm.load(Opnd::mem( + usize::BITS as u8, + string, + RUBY_OFFSET_RSTRING_AS_HEAP_PTR, + )); + // Load the address of the embedded array + // (struct RString *)(obj)->as.ary + let ary = asm.lea(Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RSTRING_AS_ARY)); + asm.csel_nz(heap_ptr, ary) +} + +fn gen_string_getbyte(asm: &mut Assembler, string: Opnd, index: Opnd) -> Opnd { + let string_ptr = get_string_ptr(asm, string); + // TODO(max): Use SIB indexing here once the backend supports it + let string_ptr = asm.add(string_ptr, index); + let byte = asm.load(Opnd::mem(8, string_ptr, 0)); + // Zero-extend the byte to 64 bits + let byte = byte.with_num_bits(64); + let byte = asm.and(byte, 0xFF.into()); + // Tag the byte + let byte = asm.lshift(byte, Opnd::UImm(1)); + asm.or(byte, Opnd::UImm(1)) +} + +fn gen_string_setbyte_fixnum(asm: &mut Assembler, string: Opnd, index: Opnd, value: Opnd) -> Opnd { + // rb_str_setbyte is not leaf, but we guard types and index ranges in HIR + asm_ccall!(asm, rb_str_setbyte, string, index, value) +} + +fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_str_buf_append, string, val) +} + +fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_jit_str_concat_codepoint, string, val) +} + +/// Generate a JIT entry that just increments exit_compilation_failure and exits +fn gen_compile_error_counter(cb: &mut CodeBlock, compile_error: &CompileError) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + gen_incr_counter(&mut asm, exit_compile_error); + gen_incr_counter(&mut asm, exit_counter_for_compile_error(compile_error)); + asm.cret(Qundef.into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(0, gc_offsets.len()); + code_ptr + }) +} + +/// Given the number of spill slots needed for a function, return the number of bytes +/// the function needs to allocate on the stack for the stack frame. +fn aligned_stack_bytes(num_slots: usize) -> usize { + // Both x86_64 and arm64 require the stack to be aligned to 16 bytes. + // Since SIZEOF_VALUE is 8 bytes, we need to round up the size to the nearest even number. + let num_slots = num_slots + (num_slots % 2); + num_slots * SIZEOF_VALUE } impl Assembler { - /// Make a C call while marking the start and end positions of it - fn ccall_with_branch(&mut self, fptr: *const u8, opnds: Vec<Opnd>, branch: &Rc<Branch>) -> Opnd { + /// Make a C call while marking the start and end positions for IseqCall + fn ccall_with_iseq_call(&mut self, fptr: *const u8, opnds: Vec<Opnd>, iseq_call: &IseqCallRef) -> Opnd { // We need to create our own branch rc objects so that we can move the closure below - let start_branch = branch.clone(); - let end_branch = branch.clone(); + let start_iseq_call = iseq_call.clone(); + let end_iseq_call = iseq_call.clone(); self.ccall_with_pos_markers( fptr, opnds, move |code_ptr, _| { - start_branch.start_addr.set(Some(code_ptr)); + start_iseq_call.start_addr.set(Some(code_ptr)); }, move |code_ptr, _| { - end_branch.end_addr.set(Some(code_ptr)); + end_iseq_call.end_addr.set(Some(code_ptr)); }, ) } } -/// Store info about an outgoing branch in a code segment +/// Store info about a JIT entry point +pub struct JITEntry { + /// Index that corresponds to [crate::hir::jit_entry_insns] + jit_entry_idx: usize, + /// Position where the entry point starts + start_addr: Cell<Option<CodePtr>>, +} + +impl JITEntry { + /// Allocate a new JITEntry + fn new(jit_entry_idx: usize) -> Rc<RefCell<Self>> { + let jit_entry = JITEntry { + jit_entry_idx, + start_addr: Cell::new(None), + }; + Rc::new(RefCell::new(jit_entry)) + } +} + +/// Store info about a JIT-to-JIT call #[derive(Debug)] -struct Branch { - /// Position where the generated code starts +pub struct IseqCall { + /// Callee ISEQ that start_addr jumps to + pub iseq: Cell<IseqPtr>, + + /// Index that corresponds to [crate::hir::jit_entry_insns] + jit_entry_idx: u32, + + /// Position where the call instruction starts start_addr: Cell<Option<CodePtr>>, - /// Position where the generated code ends (exclusive) + /// Position where the call instruction ends (exclusive) end_addr: Cell<Option<CodePtr>>, } -impl Branch { - /// Allocate a new branch - fn new() -> Rc<Self> { - Rc::new(Branch { +pub type IseqCallRef = Rc<IseqCall>; + +impl IseqCall { + /// Allocate a new IseqCall + fn new(iseq: IseqPtr, jit_entry_idx: u32) -> IseqCallRef { + let iseq_call = IseqCall { + iseq: Cell::new(iseq), start_addr: Cell::new(None), end_addr: Cell::new(None), - }) + jit_entry_idx, + }; + Rc::new(iseq_call) } - /// Regenerate a branch with a given callback + /// Regenerate a IseqCall with a given callback fn regenerate(&self, cb: &mut CodeBlock, callback: impl Fn(&mut Assembler)) { cb.with_write_ptr(self.start_addr.get().unwrap(), |cb| { let mut asm = Assembler::new(); @@ -778,3 +2861,37 @@ impl Branch { }); } } + +#[cfg(test)] +mod tests { + use crate::codegen::MAX_ISEQ_VERSIONS; + use crate::cruby::test_utils::*; + use crate::payload::*; + + #[test] + fn test_max_iseq_versions() { + eval(&format!(" + TEST = -1 + def test = TEST + + # compile and invalidate MAX+1 times + i = 0 + while i < {MAX_ISEQ_VERSIONS} + 1 + test; test # compile a version + + Object.send(:remove_const, :TEST) + TEST = i + + i += 1 + end + ")); + + // It should not exceed MAX_ISEQ_VERSIONS + let iseq = get_method_iseq("self", "test"); + let payload = get_or_create_iseq_payload(iseq); + assert_eq!(payload.versions.len(), MAX_ISEQ_VERSIONS); + + // The last call should not discard the JIT code + assert!(matches!(unsafe { payload.versions.last().unwrap().as_ref() }.status, IseqStatus::Compiled(_))); + } +} diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index c733aea99d..51faaab9c2 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -81,6 +81,7 @@ #![allow(non_camel_case_types)] // A lot of imported CRuby globals aren't all-caps #![allow(non_upper_case_globals)] +#![allow(clippy::upper_case_acronyms)] // Some of this code may not be used yet #![allow(dead_code)] @@ -88,8 +89,8 @@ #![allow(unused_imports)] use std::convert::From; -use std::ffi::{CString, CStr}; -use std::fmt::{Debug, Formatter}; +use std::ffi::{c_void, CString, CStr}; +use std::fmt::{Debug, Display, Formatter}; use std::os::raw::{c_char, c_int, c_uint}; use std::panic::{catch_unwind, UnwindSafe}; @@ -129,10 +130,12 @@ unsafe extern "C" { pub fn rb_float_new(d: f64) -> VALUE; pub fn rb_hash_empty_p(hash: VALUE) -> VALUE; - pub fn rb_yjit_str_concat_codepoint(str: VALUE, codepoint: VALUE); pub fn rb_str_setbyte(str: VALUE, index: VALUE, value: VALUE) -> VALUE; + pub fn rb_str_getbyte(str: VALUE, index: VALUE) -> VALUE; pub fn rb_vm_splat_array(flag: VALUE, ary: VALUE) -> VALUE; + pub fn rb_jit_fix_mod_fix(x: VALUE, y: VALUE) -> VALUE; pub fn rb_vm_concat_array(ary1: VALUE, ary2st: VALUE) -> VALUE; + pub fn rb_vm_get_special_object(reg_ep: *const VALUE, value_type: vm_special_object_type) -> VALUE; pub fn rb_vm_concat_to_array(ary1: VALUE, ary2st: VALUE) -> VALUE; pub fn rb_vm_defined( ec: EcPtr, @@ -143,6 +146,7 @@ unsafe extern "C" { ) -> bool; pub fn rb_vm_set_ivar_id(obj: VALUE, idx: u32, val: VALUE) -> VALUE; pub fn rb_vm_setinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, val: VALUE, ic: IVC); + pub fn rb_vm_getinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, ic: IVC) -> VALUE; pub fn rb_aliased_callable_method_entry( me: *const rb_callable_method_entry_t, ) -> *const rb_callable_method_entry_t; @@ -157,6 +161,8 @@ unsafe extern "C" { pub fn rb_vm_ic_hit_p(ic: IC, reg_ep: *const VALUE) -> bool; pub fn rb_vm_stack_canary() -> VALUE; pub fn rb_vm_push_cfunc_frame(cme: *const rb_callable_method_entry_t, recv_idx: c_int); + pub fn rb_obj_class(klass: VALUE) -> VALUE; + pub fn rb_vm_objtostring(iseq: IseqPtr, recv: VALUE, cd: *const rb_call_data) -> VALUE; } // Renames @@ -184,21 +190,7 @@ pub use rb_get_iseq_body_local_iseq as get_iseq_body_local_iseq; pub use rb_get_iseq_body_iseq_encoded as get_iseq_body_iseq_encoded; pub use rb_get_iseq_body_stack_max as get_iseq_body_stack_max; pub use rb_get_iseq_body_type as get_iseq_body_type; -pub use rb_get_iseq_flags_has_lead as get_iseq_flags_has_lead; -pub use rb_get_iseq_flags_has_opt as get_iseq_flags_has_opt; -pub use rb_get_iseq_flags_has_kw as get_iseq_flags_has_kw; -pub use rb_get_iseq_flags_has_rest as get_iseq_flags_has_rest; -pub use rb_get_iseq_flags_has_post as get_iseq_flags_has_post; -pub use rb_get_iseq_flags_has_kwrest as get_iseq_flags_has_kwrest; -pub use rb_get_iseq_flags_has_block as get_iseq_flags_has_block; -pub use rb_get_iseq_flags_ambiguous_param0 as get_iseq_flags_ambiguous_param0; -pub use rb_get_iseq_flags_accepts_no_kwarg as get_iseq_flags_accepts_no_kwarg; pub use rb_get_iseq_body_local_table_size as get_iseq_body_local_table_size; -pub use rb_get_iseq_body_param_keyword as get_iseq_body_param_keyword; -pub use rb_get_iseq_body_param_size as get_iseq_body_param_size; -pub use rb_get_iseq_body_param_lead_num as get_iseq_body_param_lead_num; -pub use rb_get_iseq_body_param_opt_num as get_iseq_body_param_opt_num; -pub use rb_get_iseq_body_param_opt_table as get_iseq_body_param_opt_table; pub use rb_get_cikw_keyword_len as get_cikw_keyword_len; pub use rb_get_cikw_keywords_idx as get_cikw_keywords_idx; pub use rb_get_call_data_ci as get_call_data_ci; @@ -206,16 +198,20 @@ pub use rb_FL_TEST as FL_TEST; pub use rb_FL_TEST_RAW as FL_TEST_RAW; pub use rb_RB_TYPE_P as RB_TYPE_P; pub use rb_BASIC_OP_UNREDEFINED_P as BASIC_OP_UNREDEFINED_P; -pub use rb_RSTRUCT_LEN as RSTRUCT_LEN; pub use rb_vm_ci_argc as vm_ci_argc; pub use rb_vm_ci_mid as vm_ci_mid; pub use rb_vm_ci_flag as vm_ci_flag; pub use rb_vm_ci_kwarg as vm_ci_kwarg; pub use rb_METHOD_ENTRY_VISI as METHOD_ENTRY_VISI; pub use rb_RCLASS_ORIGIN as RCLASS_ORIGIN; +pub use rb_vm_get_special_object as vm_get_special_object; +pub use rb_jit_fix_mod_fix as rb_fix_mod_fix; /// Helper so we can get a Rust string for insn_name() pub fn insn_name(opcode: usize) -> String { + if opcode >= VM_INSTRUCTION_SIZE.try_into().unwrap() { + return "<unknown>".into(); + } unsafe { // Look up Ruby's NULL-terminated insn name string let op_name = raw_insn_name(VALUE(opcode)); @@ -234,10 +230,12 @@ pub fn insn_len(opcode: usize) -> u32 { } } -/// Opaque iseq type for opaque iseq pointers from vm_core.h +/// We avoid using bindgen for `rb_iseq_constant_body` since its definition changes depending +/// on build configuration while we need one bindgen file that works for all configurations. +/// Use an opaque type for it instead. /// See: <https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs> #[repr(C)] -pub struct rb_iseq_t { +pub struct rb_iseq_constant_body { _data: [u8; 0], _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, } @@ -253,12 +251,31 @@ pub struct VALUE(pub usize); /// An interned string. See [ids] and methods this type. /// `0` is a sentinal value for IDs. #[repr(transparent)] -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub struct ID(pub ::std::os::raw::c_ulong); /// Pointer to an ISEQ pub type IseqPtr = *const rb_iseq_t; +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct ShapeId(pub u32); + +pub const INVALID_SHAPE_ID: ShapeId = ShapeId(rb_invalid_shape_id); + +impl ShapeId { + pub fn is_valid(self) -> bool { + self != INVALID_SHAPE_ID + } + + pub fn is_too_complex(self) -> bool { + unsafe { rb_jit_shape_too_complex_p(self.0) } + } + + pub fn is_frozen(self) -> bool { + (self.0 & SHAPE_ID_FL_FROZEN) != 0 + } +} + // Given an ISEQ pointer, convert PC to insn_idx pub fn iseq_pc_to_insn_idx(iseq: IseqPtr, pc: *mut VALUE) -> Option<u16> { let pc_zero = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; @@ -271,6 +288,41 @@ pub fn iseq_opcode_at_idx(iseq: IseqPtr, insn_idx: u32) -> u32 { unsafe { rb_iseq_opcode_at_pc(iseq, pc) as u32 } } +/// Return true if a given ISEQ is known to escape EP to the heap on entry. +/// +/// As of vm_push_frame(), EP is always equal to BP. However, after pushing +/// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP. +pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { + match unsafe { get_iseq_body_type(iseq) } { + // The EP of the <main> frame points to TOPLEVEL_BINDING + ISEQ_TYPE_MAIN | + // eval frames point to the EP of another frame or scope + ISEQ_TYPE_EVAL => true, + _ => false, + } +} + +/// Index of the local variable that has a rest parameter if any +pub fn iseq_rest_param_idx(params: &IseqParameters) -> Option<i32> { + // TODO(alan): replace with `params.rest_start` + if params.flags.has_rest() != 0 { + Some(params.opt_num + params.lead_num) + } else { + None + } +} + +/// Iterate over all existing ISEQs +pub fn for_each_iseq<F: FnMut(IseqPtr)>(mut callback: F) { + unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { + // SAFETY: points to the local below + let callback: &mut &mut dyn FnMut(IseqPtr) -> bool = unsafe { std::mem::transmute(&mut *data) }; + callback(iseq); + } + let mut data: &mut dyn FnMut(IseqPtr) = &mut callback; + unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), (&mut data) as *mut _ as *mut c_void) }; +} + /// Return a poison value to be set above the stack top to verify leafness. #[cfg(not(test))] pub fn vm_stack_canary() -> u64 { @@ -341,10 +393,27 @@ pub enum ClassRelationship { NoRelation, } +/// A print adapator for debug info about a [VALUE]. Includes info +/// the GC knows about the handle. Example: `println!("{}", value.obj_info());`. +pub struct ObjInfoPrinter(VALUE); + +impl Display for ObjInfoPrinter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + use std::mem::MaybeUninit; + const BUFFER_SIZE: usize = 0x100; + let mut buffer: MaybeUninit<[c_char; BUFFER_SIZE]> = MaybeUninit::uninit(); + let info = unsafe { + rb_raw_obj_info(buffer.as_mut_ptr().cast(), BUFFER_SIZE, self.0); + CStr::from_ptr(buffer.as_ptr().cast()).to_string_lossy() + }; + write!(f, "{info}") + } +} + impl VALUE { - /// Dump info about the value to the console similarly to rp(VALUE) - pub fn dump_info(self) { - unsafe { rb_obj_info_dump(self) } + /// Get a printer for raw debug info from `rb_obj_info()` about the value. + pub fn obj_info(self) -> ObjInfoPrinter { + ObjInfoPrinter(self) } /// Return whether the value is truthy or falsy in Ruby -- only nil and false are falsy. @@ -371,6 +440,11 @@ impl VALUE { !self.special_const_p() } + /// Shareability between ractors. `RB_OBJ_SHAREABLE_P()`. + pub fn shareable_p(self) -> bool { + (self.builtin_flags() & RUBY_FL_SHAREABLE as usize) != 0 + } + /// Return true if the value is a Ruby Fixnum (immediate-size integer) pub fn fixnum_p(self) -> bool { let VALUE(cval) = self; @@ -391,6 +465,16 @@ impl VALUE { self.static_sym_p() || self.dynamic_sym_p() } + pub fn instance_can_have_singleton_class(self) -> bool { + if self == unsafe { rb_cInteger } || self == unsafe { rb_cFloat } || + self == unsafe { rb_cSymbol } || self == unsafe { rb_cNilClass } || + self == unsafe { rb_cTrueClass } || self == unsafe { rb_cFalseClass } { + + return false + } + true + } + /// Return true for a static (non-heap) Ruby symbol (RB_STATIC_SYM_P) pub fn static_sym_p(self) -> bool { let VALUE(cval) = self; @@ -438,8 +522,11 @@ impl VALUE { pub fn class_of(self) -> VALUE { if !self.special_const_p() { let builtin_type = self.builtin_type(); - assert_ne!(builtin_type, RUBY_T_NONE, "ZJIT should only see live objects"); - assert_ne!(builtin_type, RUBY_T_MOVED, "ZJIT should only see live objects"); + assert!( + builtin_type != RUBY_T_NONE && builtin_type != RUBY_T_MOVED, + "ZJIT saw a dead object. T_type={builtin_type}, {}", + self.obj_info() + ); } unsafe { rb_yarv_class_of(self) } @@ -477,29 +564,20 @@ impl VALUE { unsafe { rb_obj_frozen_p(self) != VALUE(0) } } - pub fn shape_too_complex(self) -> bool { - unsafe { rb_shape_obj_too_complex_p(self) } + pub fn shape_id_of(self) -> ShapeId { + ShapeId(unsafe { rb_obj_shape_id(self) }) } - pub fn shape_id_of(self) -> u32 { - unsafe { rb_obj_shape_id(self) } - } - - pub fn shape_of(self) -> *mut rb_shape { + pub fn embedded_p(self) -> bool { unsafe { - let shape = rb_shape_lookup(self.shape_id_of()); - - if shape.is_null() { - panic!("Shape should not be null"); - } else { - shape - } + FL_TEST_RAW(self, VALUE(ROBJECT_HEAP as usize)) == VALUE(0) } } - pub fn embedded_p(self) -> bool { + pub fn struct_embedded_p(self) -> bool { unsafe { - FL_TEST_RAW(self, VALUE(ROBJECT_EMBED as usize)) != VALUE(0) + RB_TYPE_P(self, RUBY_T_STRUCT) && + FL_TEST_RAW(self, VALUE(RSTRUCT_EMBED_LEN_MASK)) != VALUE(0) } } @@ -573,6 +651,11 @@ impl VALUE { ptr } + pub fn cme_p(self) -> bool { + if self == VALUE(0) { return false; } + unsafe { rb_IMEMO_TYPE_P(self, imemo_ment) == 1 } + } + /// Assert that `self` is a method entry in debug builds pub fn as_cme(self) -> *const rb_callable_method_entry_t { let ptr: *const rb_callable_method_entry_t = self.as_ptr(); @@ -597,6 +680,29 @@ impl VALUE { let k: isize = item.wrapping_add(item.wrapping_add(1)); VALUE(k as usize) } + + /// Call the write barrier after separately writing val to self. + pub fn write_barrier(self, val: VALUE) { + // rb_gc_writebarrier() asserts it is not called with a special constant + if !val.special_const_p() { + unsafe { rb_gc_writebarrier(self, val) }; + } + } +} + +pub type IseqParameters = rb_iseq_constant_body_rb_iseq_parameters; + +/// Extension trait to enable method calls on [`IseqPtr`] +pub trait IseqAccess { + unsafe fn params<'a>(self) -> &'a IseqParameters; +} + +impl IseqAccess for IseqPtr { + /// Get a description of the ISEQ's signature. Analogous to `ISEQ_BODY(iseq)->param` in C. + unsafe fn params<'a>(self) -> &'a IseqParameters { + use crate::cast::IntoUsize; + unsafe { &*((*self).body.byte_add(ISEQ_BODY_OFFSET_PARAM.to_usize()) as *const IseqParameters) } + } } impl From<IseqPtr> for VALUE { @@ -682,16 +788,22 @@ pub fn rust_str_to_ruby(str: &str) -> VALUE { unsafe { rb_utf8_str_new(str.as_ptr() as *const _, str.len() as i64) } } -/// Produce a Ruby symbol from a Rust string slice -pub fn rust_str_to_sym(str: &str) -> VALUE { +/// Produce a Ruby ID from a Rust string slice +pub fn rust_str_to_id(str: &str) -> ID { let c_str = CString::new(str).unwrap(); let c_ptr: *const c_char = c_str.as_ptr(); - unsafe { rb_id2sym(rb_intern(c_ptr)) } + unsafe { rb_intern(c_ptr) } +} + +/// Produce a Ruby symbol from a Rust string slice +pub fn rust_str_to_sym(str: &str) -> VALUE { + let id = rust_str_to_id(str); + unsafe { rb_id2sym(id) } } /// Produce an owned Rust String from a C char pointer pub fn cstr_to_rust_string(c_char_ptr: *const c_char) -> Option<String> { - assert!(c_char_ptr != std::ptr::null()); + assert!(!c_char_ptr.is_null()); let c_str: &CStr = unsafe { CStr::from_ptr(c_char_ptr) }; @@ -702,29 +814,32 @@ pub fn cstr_to_rust_string(c_char_ptr: *const c_char) -> Option<String> { } pub fn iseq_name(iseq: IseqPtr) -> String { + if iseq.is_null() { + return "<NULL>".to_string(); + } let iseq_label = unsafe { rb_iseq_label(iseq) }; if iseq_label == Qnil { "None".to_string() } else { - ruby_str_to_rust(iseq_label) + ruby_str_to_rust_string(iseq_label) } } // Location is the file defining the method, colon, method name. // Filenames are sometimes internal strings supplied to eval, // so be careful with them. -pub fn iseq_get_location(iseq: IseqPtr, pos: u16) -> String { +pub fn iseq_get_location(iseq: IseqPtr, pos: u32) -> String { let iseq_path = unsafe { rb_iseq_path(iseq) }; let iseq_lineno = unsafe { rb_iseq_line_no(iseq, pos as usize) }; let mut s = iseq_name(iseq); - s.push_str("@"); + s.push('@'); if iseq_path == Qnil { s.push_str("None"); } else { - s.push_str(&ruby_str_to_rust(iseq_path)); + s.push_str(&ruby_str_to_rust_string(iseq_path)); } - s.push_str(":"); + s.push(':'); s.push_str(&iseq_lineno.to_string()); s } @@ -733,14 +848,26 @@ pub fn iseq_get_location(iseq: IseqPtr, pos: u16) -> String { // Convert a CRuby UTF-8-encoded RSTRING into a Rust string. // This should work fine on ASCII strings and anything else // that is considered legal UTF-8, including embedded nulls. -fn ruby_str_to_rust(v: VALUE) -> String { +fn ruby_str_to_rust_string(v: VALUE) -> String { let str_ptr = unsafe { rb_RSTRING_PTR(v) } as *mut u8; let str_len: usize = unsafe { rb_RSTRING_LEN(v) }.try_into().unwrap(); let str_slice: &[u8] = unsafe { std::slice::from_raw_parts(str_ptr, str_len) }; - match String::from_utf8(str_slice.to_vec()) { - Ok(utf8) => utf8, - Err(_) => String::new(), - } + String::from_utf8(str_slice.to_vec()).unwrap_or_default() +} + +pub fn ruby_sym_to_rust_string(v: VALUE) -> String { + let ruby_str = unsafe { rb_sym2str(v) }; + ruby_str_to_rust_string(ruby_str) +} + +pub fn ruby_call_method_id(cd: *const rb_call_data) -> ID { + let call_info = unsafe { rb_get_call_data_ci(cd) }; + unsafe { rb_vm_ci_mid(call_info) } +} + +pub fn ruby_call_method_name(cd: *const rb_call_data) -> String { + let mid = ruby_call_method_id(cd); + mid.contents_lossy().to_string() } /// A location in Rust code for integrating with debugging facilities defined in C. @@ -796,7 +923,15 @@ where let line = loc.line; let mut recursive_lock_level: c_uint = 0; - unsafe { rb_zjit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; + unsafe { rb_jit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; + // Ensure GC is off while we have the VM lock because: + // 1. We create many transient Rust collections that hold VALUEs during compilation. + // It's extremely tricky to properly marked and reference update these, not to + // mention the overhead and ergonomics issues. + // 2. If we yield to the GC while compiling, it re-enters our mark and update functions. + // This breaks `&mut` exclusivity since mark functions derive fresh `&mut` from statics + // while there is a stack frame below it that has an overlapping `&mut`. That's UB. + let gc_disabled_pre_call = unsafe { rb_gc_disable() }.test(); let ret = match catch_unwind(func) { Ok(result) => result, @@ -816,7 +951,12 @@ where } }; - unsafe { rb_zjit_vm_unlock(&mut recursive_lock_level, file, line) }; + unsafe { + if !gc_disabled_pre_call { + rb_gc_enable(); + } + rb_jit_vm_unlock(&mut recursive_lock_level, file, line); + }; ret } @@ -853,9 +993,11 @@ pub fn rb_bug_panic_hook() { // You may also use ZJIT_RB_BUG=1 to trigger this on dev builds. if release_build || env::var("ZJIT_RB_BUG").is_ok() { // Abort with rb_bug(). It has a length limit on the message. - let panic_message = &format!("{}", panic_info)[..]; + let panic_message = &format!("{panic_info}")[..]; let len = std::cmp::min(0x100, panic_message.len()) as c_int; unsafe { rb_bug(b"ZJIT: %*s\0".as_ref().as_ptr() as *const c_char, len, panic_message.as_ptr()); } + } else { + eprintln!("note: run with `ZJIT_RB_BUG=1` environment variable to display a Ruby backtrace"); } })); } @@ -875,8 +1017,9 @@ mod manual_defs { use super::*; pub const SIZEOF_VALUE: usize = 8; + pub const BITS_PER_BYTE: usize = 8; pub const SIZEOF_VALUE_I32: i32 = SIZEOF_VALUE as i32; - pub const VALUE_BITS: u8 = 8 * SIZEOF_VALUE as u8; + pub const VALUE_BITS: u8 = BITS_PER_BYTE as u8 * SIZEOF_VALUE as u8; pub const RUBY_LONG_MIN: isize = std::os::raw::c_long::MIN as isize; pub const RUBY_LONG_MAX: isize = std::os::raw::c_long::MAX as isize; @@ -929,12 +1072,6 @@ mod manual_defs { pub const RUBY_OFFSET_CFP_JIT_RETURN: i32 = 48; pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 56; - // Constants from rb_execution_context_t vm_core.h - pub const RUBY_OFFSET_EC_CFP: i32 = 16; - pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: i32 = 32; // rb_atomic_t (u32) - pub const RUBY_OFFSET_EC_INTERRUPT_MASK: i32 = 36; // rb_atomic_t (u32) - pub const RUBY_OFFSET_EC_THREAD_PTR: i32 = 48; - // Constants from rb_thread_t in vm_core.h pub const RUBY_OFFSET_THREAD_SELF: i32 = 16; @@ -948,7 +1085,7 @@ pub use manual_defs::*; pub mod test_utils { use std::{ptr::null, sync::Once}; - use crate::{options::init_options, state::rb_zjit_enabled_p, state::ZJITState}; + use crate::{options::{rb_zjit_call_threshold, rb_zjit_prepare_options, set_call_threshold, DEFAULT_CALL_THRESHOLD}, state::{rb_zjit_entry, ZJITState}}; use super::*; @@ -970,8 +1107,14 @@ pub mod test_utils { // <https://github.com/Shopify/zjit/pull/37>, though let mut var: VALUE = Qnil; ruby_init_stack(&mut var as *mut VALUE as *mut _); + rb_zjit_prepare_options(); // enable `#with_jit` on builtins ruby_init(); + // The default rb_zjit_profile_threshold is too high, so lower it for HIR tests. + if rb_zjit_call_threshold == DEFAULT_CALL_THRESHOLD { + set_call_threshold(2); + } + // Pass command line options so the VM loads core library methods defined in // ruby such as from `kernel.rb`. // We drive ZJIT manually in tests, so disable heuristic compilation triggers. @@ -985,15 +1128,15 @@ pub mod test_utils { } // Set up globals for convenience - ZJITState::init(init_options()); + let zjit_entry = ZJITState::init(); // Enable zjit_* instructions - unsafe { rb_zjit_enabled_p = true; } + unsafe { rb_zjit_entry = zjit_entry; } } /// Make sure the Ruby VM is set up and run a given callback with rb_protect() pub fn with_rubyvm<T>(mut func: impl FnMut() -> T) -> T { - RUBY_VM_INIT.call_once(|| boot_rubyvm()); + RUBY_VM_INIT.call_once(boot_rubyvm); // Set up a callback wrapper to store a return value let mut result: Option<T> = None; @@ -1042,9 +1185,25 @@ pub mod test_utils { }) } - /// Get the ISeq of a specified method - pub fn get_method_iseq(name: &str) -> *const rb_iseq_t { - let wrapped_iseq = eval(&format!("RubyVM::InstructionSequence.of(method(:{}))", name)); + /// Get the #inspect of a given Ruby program in Rust string + pub fn inspect(program: &str) -> String { + let inspect = format!("({program}).inspect"); + ruby_str_to_rust_string(eval(&inspect)) + } + + /// Get IseqPtr for a specified method + pub fn get_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { + get_proc_iseq(&format!("{}.method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified instance method + pub fn get_instance_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { + get_proc_iseq(&format!("{}.instance_method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified Proc object + pub fn get_proc_iseq(obj: &str) -> *const rb_iseq_t { + let wrapped_iseq = eval(&format!("RubyVM::InstructionSequence.of({obj})")); unsafe { rb_iseqw_to_iseq(wrapped_iseq) } } @@ -1073,7 +1232,7 @@ pub mod test_utils { if line.len() > spaces { unindented.extend_from_slice(&line.as_bytes()[spaces..]); } else { - unindented.extend_from_slice(&line.as_bytes()); + unindented.extend_from_slice(line.as_bytes()); } } String::from_utf8(unindented).unwrap() @@ -1161,6 +1320,19 @@ pub fn get_class_name(class: VALUE) -> String { }).unwrap_or_else(|| "Unknown".to_string()) } +pub fn class_has_leaf_allocator(class: VALUE) -> bool { + // empty_hash_alloc + if class == unsafe { rb_cHash } { return true; } + // empty_ary_alloc + if class == unsafe { rb_cArray } { return true; } + // empty_str_alloc + if class == unsafe { rb_cString } { return true; } + // rb_reg_s_alloc + if class == unsafe { rb_cRegexp } { return true; } + // rb_class_allocate_instance + unsafe { rb_zjit_class_has_default_allocator(class) } +} + /// Interned ID values for Ruby symbols and method names. /// See [type@crate::cruby::ID] and usages outside of ZJIT. pub(crate) mod ids { @@ -1196,11 +1368,34 @@ pub(crate) mod ids { name: NULL content: b"" name: respond_to_missing content: b"respond_to_missing?" name: eq content: b"==" + name: string_eq content: b"String#==" name: include_p content: b"include?" name: to_ary name: to_s name: compile name: eval + name: plus content: b"+" + name: minus content: b"-" + name: mult content: b"*" + name: div content: b"/" + name: modulo content: b"%" + name: neq content: b"!=" + name: lt content: b"<" + name: le content: b"<=" + name: gt content: b">" + name: ge content: b">=" + name: and content: b"&" + name: or content: b"|" + name: xor content: b"^" + name: freeze + name: minusat content: b"-@" + name: aref content: b"[]" + name: len + name: _as_heap + name: thread_ptr + name: self_ content: b"self" + name: rb_ivar_get_at_no_ractor_check + name: _shape_id } /// Get an CRuby `ID` to an interned string, e.g. a particular method name. diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 10dc406aca..15533180da 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1,6 +1,143 @@ /* automatically generated by rust-bindgen 0.71.1 */ #[repr(C)] +#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct __BindgenBitfieldUnit<Storage> { + storage: Storage, +} +impl<Storage> __BindgenBitfieldUnit<Storage> { + #[inline] + pub const fn new(storage: Storage) -> Self { + Self { storage } + } +} +impl<Storage> __BindgenBitfieldUnit<Storage> +where + Storage: AsRef<[u8]> + AsMut<[u8]>, +{ + #[inline] + fn extract_bit(byte: u8, index: usize) -> bool { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + byte & mask == mask + } + #[inline] + pub fn get_bit(&self, index: usize) -> bool { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = self.storage.as_ref()[byte_index]; + Self::extract_bit(byte, index) + } + #[inline] + pub unsafe fn raw_get_bit(this: *const Self, index: usize) -> bool { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = *(core::ptr::addr_of!((*this).storage) as *const u8).offset(byte_index as isize); + Self::extract_bit(byte, index) + } + #[inline] + fn change_bit(byte: u8, index: usize, val: bool) -> u8 { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + if val { + byte | mask + } else { + byte & !mask + } + } + #[inline] + pub fn set_bit(&mut self, index: usize, val: bool) { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = &mut self.storage.as_mut()[byte_index]; + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub unsafe fn raw_set_bit(this: *mut Self, index: usize, val: bool) { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = + (core::ptr::addr_of_mut!((*this).storage) as *mut u8).offset(byte_index as isize); + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub fn get(&self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if self.get_bit(i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub unsafe fn raw_get(this: *const Self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if Self::raw_get_bit(this, i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub fn set(&mut self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + self.set_bit(index + bit_offset, val_bit_is_set); + } + } + #[inline] + pub unsafe fn raw_set(this: *mut Self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + Self::raw_set_bit(this, index + bit_offset, val_bit_is_set); + } + } +} +#[repr(C)] #[derive(Default)] pub struct __IncompleteArrayField<T>(::std::marker::PhantomData<T>, [T; 0]); impl<T> __IncompleteArrayField<T> { @@ -30,6 +167,54 @@ impl<T> ::std::fmt::Debug for __IncompleteArrayField<T> { fmt.write_str("__IncompleteArrayField") } } +#[repr(C)] +pub struct __BindgenUnionField<T>(::std::marker::PhantomData<T>); +impl<T> __BindgenUnionField<T> { + #[inline] + pub const fn new() -> Self { + __BindgenUnionField(::std::marker::PhantomData) + } + #[inline] + pub unsafe fn as_ref(&self) -> &T { + ::std::mem::transmute(self) + } + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut T { + ::std::mem::transmute(self) + } +} +impl<T> ::std::default::Default for __BindgenUnionField<T> { + #[inline] + fn default() -> Self { + Self::new() + } +} +impl<T> ::std::clone::Clone for __BindgenUnionField<T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} +impl<T> ::std::marker::Copy for __BindgenUnionField<T> {} +impl<T> ::std::fmt::Debug for __BindgenUnionField<T> { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + fmt.write_str("__BindgenUnionField") + } +} +impl<T> ::std::hash::Hash for __BindgenUnionField<T> { + fn hash<H: ::std::hash::Hasher>(&self, _state: &mut H) {} +} +impl<T> ::std::cmp::PartialEq for __BindgenUnionField<T> { + fn eq(&self, _other: &__BindgenUnionField<T>) -> bool { + true + } +} +impl<T> ::std::cmp::Eq for __BindgenUnionField<T> {} +pub const ONIG_OPTION_IGNORECASE: u32 = 1; +pub const ONIG_OPTION_EXTEND: u32 = 2; +pub const ONIG_OPTION_MULTILINE: u32 = 4; +pub const ARG_ENCODING_FIXED: u32 = 16; +pub const ARG_ENCODING_NONE: u32 = 32; pub const INTEGER_REDEFINED_OP_FLAG: u32 = 1; pub const FLOAT_REDEFINED_OP_FLAG: u32 = 2; pub const STRING_REDEFINED_OP_FLAG: u32 = 4; @@ -42,6 +227,7 @@ pub const NIL_REDEFINED_OP_FLAG: u32 = 512; pub const TRUE_REDEFINED_OP_FLAG: u32 = 1024; pub const FALSE_REDEFINED_OP_FLAG: u32 = 2048; pub const PROC_REDEFINED_OP_FLAG: u32 = 4096; +pub const VM_KW_SPECIFIED_BITS_MAX: u32 = 31; pub const VM_ENV_DATA_SIZE: u32 = 3; pub const VM_ENV_DATA_INDEX_ME_CREF: i32 = -2; pub const VM_ENV_DATA_INDEX_SPECVAL: i32 = -1; @@ -98,13 +284,12 @@ pub const RUBY_FL_USHIFT: ruby_fl_ushift = 12; pub type ruby_fl_ushift = u32; pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32; pub const RUBY_FL_PROMOTED: ruby_fl_type = 32; -pub const RUBY_FL_UNUSED6: ruby_fl_type = 64; +pub const RUBY_FL_USERPRIV0: ruby_fl_type = 64; pub const RUBY_FL_FINALIZE: ruby_fl_type = 128; -pub const RUBY_FL_TAINT: ruby_fl_type = 0; +pub const RUBY_FL_EXIVAR: ruby_fl_type = 0; pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256; -pub const RUBY_FL_UNTRUSTED: ruby_fl_type = 0; -pub const RUBY_FL_UNUSED9: ruby_fl_type = 512; -pub const RUBY_FL_EXIVAR: ruby_fl_type = 1024; +pub const RUBY_FL_WEAK_REFERENCE: ruby_fl_type = 512; +pub const RUBY_FL_UNUSED10: ruby_fl_type = 1024; pub const RUBY_FL_FREEZE: ruby_fl_type = 2048; pub const RUBY_FL_USER0: ruby_fl_type = 4096; pub const RUBY_FL_USER1: ruby_fl_type = 8192; @@ -152,10 +337,21 @@ pub const RARRAY_EMBED_LEN_MASK: ruby_rarray_flags = 4161536; pub type ruby_rarray_flags = u32; pub const RARRAY_EMBED_LEN_SHIFT: ruby_rarray_consts = 15; pub type ruby_rarray_consts = u32; -pub const RMODULE_IS_REFINEMENT: ruby_rmodule_flags = 32768; +pub const RMODULE_IS_REFINEMENT: ruby_rmodule_flags = 8192; pub type ruby_rmodule_flags = u32; -pub const ROBJECT_EMBED: ruby_robject_flags = 8192; +pub const ROBJECT_HEAP: ruby_robject_flags = 65536; pub type ruby_robject_flags = u32; +pub type rb_event_flag_t = u32; +pub type rb_block_call_func = ::std::option::Option< + unsafe extern "C" fn( + yielded_arg: VALUE, + callback_arg: VALUE, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + blockarg: VALUE, + ) -> VALUE, +>; +pub type rb_block_call_func_t = rb_block_call_func; pub const RUBY_ENCODING_INLINE_MAX: ruby_encoding_consts = 127; pub const RUBY_ENCODING_SHIFT: ruby_encoding_consts = 22; pub const RUBY_ENCODING_MASK: ruby_encoding_consts = 532676608; @@ -194,22 +390,23 @@ pub const BOP_NIL_P: ruby_basic_operators = 15; pub const BOP_SUCC: ruby_basic_operators = 16; pub const BOP_GT: ruby_basic_operators = 17; pub const BOP_GE: ruby_basic_operators = 18; -pub const BOP_NOT: ruby_basic_operators = 19; -pub const BOP_NEQ: ruby_basic_operators = 20; -pub const BOP_MATCH: ruby_basic_operators = 21; -pub const BOP_FREEZE: ruby_basic_operators = 22; -pub const BOP_UMINUS: ruby_basic_operators = 23; -pub const BOP_MAX: ruby_basic_operators = 24; -pub const BOP_MIN: ruby_basic_operators = 25; -pub const BOP_HASH: ruby_basic_operators = 26; -pub const BOP_CALL: ruby_basic_operators = 27; -pub const BOP_AND: ruby_basic_operators = 28; -pub const BOP_OR: ruby_basic_operators = 29; -pub const BOP_CMP: ruby_basic_operators = 30; -pub const BOP_DEFAULT: ruby_basic_operators = 31; -pub const BOP_PACK: ruby_basic_operators = 32; -pub const BOP_INCLUDE_P: ruby_basic_operators = 33; -pub const BOP_LAST_: ruby_basic_operators = 34; +pub const BOP_GTGT: ruby_basic_operators = 19; +pub const BOP_NOT: ruby_basic_operators = 20; +pub const BOP_NEQ: ruby_basic_operators = 21; +pub const BOP_MATCH: ruby_basic_operators = 22; +pub const BOP_FREEZE: ruby_basic_operators = 23; +pub const BOP_UMINUS: ruby_basic_operators = 24; +pub const BOP_MAX: ruby_basic_operators = 25; +pub const BOP_MIN: ruby_basic_operators = 26; +pub const BOP_HASH: ruby_basic_operators = 27; +pub const BOP_CALL: ruby_basic_operators = 28; +pub const BOP_AND: ruby_basic_operators = 29; +pub const BOP_OR: ruby_basic_operators = 30; +pub const BOP_CMP: ruby_basic_operators = 31; +pub const BOP_DEFAULT: ruby_basic_operators = 32; +pub const BOP_PACK: ruby_basic_operators = 33; +pub const BOP_INCLUDE_P: ruby_basic_operators = 34; +pub const BOP_LAST_: ruby_basic_operators = 35; pub type ruby_basic_operators = u32; pub type rb_serial_t = ::std::os::raw::c_ulonglong; #[repr(C)] @@ -226,12 +423,25 @@ pub const imemo_memo: imemo_type = 5; pub const imemo_ment: imemo_type = 6; pub const imemo_iseq: imemo_type = 7; pub const imemo_tmpbuf: imemo_type = 8; -pub const imemo_ast: imemo_type = 9; -pub const imemo_parser_strterm: imemo_type = 10; -pub const imemo_callinfo: imemo_type = 11; -pub const imemo_callcache: imemo_type = 12; -pub const imemo_constcache: imemo_type = 13; +pub const imemo_callinfo: imemo_type = 10; +pub const imemo_callcache: imemo_type = 11; +pub const imemo_constcache: imemo_type = 12; +pub const imemo_fields: imemo_type = 13; pub type imemo_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vm_ifunc_argc { + pub min: ::std::os::raw::c_int, + pub max: ::std::os::raw::c_int, +} +#[repr(C)] +pub struct vm_ifunc { + pub flags: VALUE, + pub svar_lep: *mut VALUE, + pub func: rb_block_call_func_t, + pub data: *const ::std::os::raw::c_void, + pub argc: vm_ifunc_argc, +} pub const METHOD_VISI_UNDEF: rb_method_visibility_t = 0; pub const METHOD_VISI_PUBLIC: rb_method_visibility_t = 1; pub const METHOD_VISI_PRIVATE: rb_method_visibility_t = 2; @@ -269,6 +479,7 @@ pub const VM_METHOD_TYPE_OPTIMIZED: rb_method_type_t = 9; pub const VM_METHOD_TYPE_MISSING: rb_method_type_t = 10; pub const VM_METHOD_TYPE_REFINED: rb_method_type_t = 11; pub type rb_method_type_t = u32; +pub type rb_iseq_t = rb_iseq_struct; pub type rb_cfunc_t = ::std::option::Option<unsafe extern "C" fn() -> VALUE>; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -291,7 +502,22 @@ pub const OPTIMIZED_METHOD_TYPE_STRUCT_AREF: method_optimized_type = 3; pub const OPTIMIZED_METHOD_TYPE_STRUCT_ASET: method_optimized_type = 4; pub const OPTIMIZED_METHOD_TYPE__MAX: method_optimized_type = 5; pub type method_optimized_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_position_struct { + pub lineno: ::std::os::raw::c_int, + pub column: ::std::os::raw::c_int, +} +pub type rb_code_position_t = rb_code_position_struct; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_location_struct { + pub beg_pos: rb_code_position_t, + pub end_pos: rb_code_position_t, +} +pub type rb_code_location_t = rb_code_location_struct; pub type rb_num_t = ::std::os::raw::c_ulong; +pub type rb_snum_t = ::std::os::raw::c_long; pub const RUBY_TAG_NONE: ruby_tag_type = 0; pub const RUBY_TAG_RETURN: ruby_tag_type = 1; pub const RUBY_TAG_BREAK: ruby_tag_type = 2; @@ -310,8 +536,6 @@ pub type ruby_vm_throw_flags = u32; pub struct iseq_inline_constant_cache_entry { pub flags: VALUE, pub value: VALUE, - pub _unused1: VALUE, - pub _unused2: VALUE, pub ic_cref: *const rb_cref_t, } #[repr(C)] @@ -322,7 +546,7 @@ pub struct iseq_inline_constant_cache { } #[repr(C)] pub struct iseq_inline_iv_cache_entry { - pub value: usize, + pub value: u64, pub iv_set_name: ID, } #[repr(C)] @@ -330,6 +554,23 @@ pub struct iseq_inline_iv_cache_entry { pub struct iseq_inline_cvar_cache_entry { pub entry: *mut rb_cvar_class_tbl_entry, } +#[repr(C)] +#[repr(align(8))] +#[derive(Copy, Clone)] +pub struct iseq_inline_storage_entry { + pub _bindgen_opaque_blob: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_location_struct { + pub pathobj: VALUE, + pub base_label: VALUE, + pub label: VALUE, + pub first_lineno: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub code_location: rb_code_location_t, +} +pub type rb_iseq_location_t = rb_iseq_location_struct; +pub type iseq_bits_t = usize; pub const ISEQ_TYPE_TOP: rb_iseq_type = 0; pub const ISEQ_TYPE_METHOD: rb_iseq_type = 1; pub const ISEQ_TYPE_BLOCK: rb_iseq_type = 2; @@ -345,9 +586,577 @@ pub const BUILTIN_ATTR_SINGLE_NOARG_LEAF: rb_builtin_attr = 2; pub const BUILTIN_ATTR_INLINE_BLOCK: rb_builtin_attr = 4; pub const BUILTIN_ATTR_C_TRACE: rb_builtin_attr = 8; pub type rb_builtin_attr = u32; +pub type rb_jit_func_t = ::std::option::Option< + unsafe extern "C" fn( + arg1: *mut rb_execution_context_struct, + arg2: *mut rb_control_frame_struct, + ) -> VALUE, +>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters { + pub flags: rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1, + pub size: ::std::os::raw::c_uint, + pub lead_num: ::std::os::raw::c_int, + pub opt_num: ::std::os::raw::c_int, + pub rest_start: ::std::os::raw::c_int, + pub post_start: ::std::os::raw::c_int, + pub post_num: ::std::os::raw::c_int, + pub block_start: ::std::os::raw::c_int, + pub opt_table: *const VALUE, + pub keyword: *const rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword, +} #[repr(C)] +#[repr(align(4))] #[derive(Debug, Copy, Clone)] -pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword { +pub struct rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 2usize]>, + pub __bindgen_padding_0: u16, +} +impl rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + #[inline] + pub fn has_lead(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_lead(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_lead_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_lead_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_opt(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_opt(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_opt_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_opt_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_post(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_post(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_post_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 3usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_post_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 3usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kw(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kw(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kw_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 4usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kw_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 4usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 5usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 5usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(6usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(6usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 6usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 6usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ambiguous_param0(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(7usize, 1u8) as u32) } + } + #[inline] + pub fn set_ambiguous_param0(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(7usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ambiguous_param0_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 7usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ambiguous_param0_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 7usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn accepts_no_kwarg(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(8usize, 1u8) as u32) } + } + #[inline] + pub fn set_accepts_no_kwarg(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(8usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn accepts_no_kwarg_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 8usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_accepts_no_kwarg_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 8usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ruby2_keywords(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(9usize, 1u8) as u32) } + } + #[inline] + pub fn set_ruby2_keywords(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(9usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ruby2_keywords_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 9usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ruby2_keywords_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 9usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(10usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(10usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 10usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 10usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(11usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(11usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 11usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 11usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn use_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(12usize, 1u8) as u32) } + } + #[inline] + pub fn set_use_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(12usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn use_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 12usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_use_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 12usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn forwardable(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(13usize, 1u8) as u32) } + } + #[inline] + pub fn set_forwardable(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(13usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn forwardable_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 13usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_forwardable_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 13usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + has_lead: ::std::os::raw::c_uint, + has_opt: ::std::os::raw::c_uint, + has_rest: ::std::os::raw::c_uint, + has_post: ::std::os::raw::c_uint, + has_kw: ::std::os::raw::c_uint, + has_kwrest: ::std::os::raw::c_uint, + has_block: ::std::os::raw::c_uint, + ambiguous_param0: ::std::os::raw::c_uint, + accepts_no_kwarg: ::std::os::raw::c_uint, + ruby2_keywords: ::std::os::raw::c_uint, + anon_rest: ::std::os::raw::c_uint, + anon_kwrest: ::std::os::raw::c_uint, + use_block: ::std::os::raw::c_uint, + forwardable: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 2usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 2usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let has_lead: u32 = unsafe { ::std::mem::transmute(has_lead) }; + has_lead as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let has_opt: u32 = unsafe { ::std::mem::transmute(has_opt) }; + has_opt as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let has_rest: u32 = unsafe { ::std::mem::transmute(has_rest) }; + has_rest as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let has_post: u32 = unsafe { ::std::mem::transmute(has_post) }; + has_post as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let has_kw: u32 = unsafe { ::std::mem::transmute(has_kw) }; + has_kw as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let has_kwrest: u32 = unsafe { ::std::mem::transmute(has_kwrest) }; + has_kwrest as u64 + }); + __bindgen_bitfield_unit.set(6usize, 1u8, { + let has_block: u32 = unsafe { ::std::mem::transmute(has_block) }; + has_block as u64 + }); + __bindgen_bitfield_unit.set(7usize, 1u8, { + let ambiguous_param0: u32 = unsafe { ::std::mem::transmute(ambiguous_param0) }; + ambiguous_param0 as u64 + }); + __bindgen_bitfield_unit.set(8usize, 1u8, { + let accepts_no_kwarg: u32 = unsafe { ::std::mem::transmute(accepts_no_kwarg) }; + accepts_no_kwarg as u64 + }); + __bindgen_bitfield_unit.set(9usize, 1u8, { + let ruby2_keywords: u32 = unsafe { ::std::mem::transmute(ruby2_keywords) }; + ruby2_keywords as u64 + }); + __bindgen_bitfield_unit.set(10usize, 1u8, { + let anon_rest: u32 = unsafe { ::std::mem::transmute(anon_rest) }; + anon_rest as u64 + }); + __bindgen_bitfield_unit.set(11usize, 1u8, { + let anon_kwrest: u32 = unsafe { ::std::mem::transmute(anon_kwrest) }; + anon_kwrest as u64 + }); + __bindgen_bitfield_unit.set(12usize, 1u8, { + let use_block: u32 = unsafe { ::std::mem::transmute(use_block) }; + use_block as u64 + }); + __bindgen_bitfield_unit.set(13usize, 1u8, { + let forwardable: u32 = unsafe { ::std::mem::transmute(forwardable) }; + forwardable as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword { pub num: ::std::os::raw::c_int, pub required_num: ::std::os::raw::c_int, pub bits_start: ::std::os::raw::c_int, @@ -355,7 +1164,217 @@ pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword { pub table: *const ID, pub default_values: *mut VALUE, } +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_iseq_insn_info { + pub body: *const iseq_insn_info_entry, + pub positions: *mut ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub succ_index_table: *mut succ_index_table, +} +pub const lvar_uninitialized: rb_iseq_constant_body_lvar_state = 0; +pub const lvar_initialized: rb_iseq_constant_body_lvar_state = 1; +pub const lvar_reassigned: rb_iseq_constant_body_lvar_state = 2; +pub type rb_iseq_constant_body_lvar_state = u32; +#[repr(C)] +pub struct rb_iseq_constant_body__bindgen_ty_1 { + pub flip_count: rb_snum_t, + pub script_lines: VALUE, + pub coverage: VALUE, + pub pc2branchindex: VALUE, + pub original_iseq: *mut VALUE, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union rb_iseq_constant_body__bindgen_ty_2 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +pub struct rb_iseq_struct { + pub flags: VALUE, + pub wrapper: VALUE, + pub body: *mut rb_iseq_constant_body, + pub aux: rb_iseq_struct__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1 { + pub compile_data: __BindgenUnionField<*mut iseq_compile_data>, + pub loader: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_1>, + pub exec: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_2>, + pub bindgen_union_field: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_1 { + pub obj: VALUE, + pub index: ::std::os::raw::c_int, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_2 { + pub local_hooks_cnt: ::std::os::raw::c_uint, + pub global_trace_events: rb_event_flag_t, +} +#[repr(C)] +pub struct rb_captured_block { + pub self_: VALUE, + pub ep: *const VALUE, + pub code: rb_captured_block__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_captured_block__bindgen_ty_1 { + pub iseq: __BindgenUnionField<*const rb_iseq_t>, + pub ifunc: __BindgenUnionField<*const vm_ifunc>, + pub val: __BindgenUnionField<VALUE>, + pub bindgen_union_field: u64, +} +pub const block_type_iseq: rb_block_type = 0; +pub const block_type_ifunc: rb_block_type = 1; +pub const block_type_symbol: rb_block_type = 2; +pub const block_type_proc: rb_block_type = 3; +pub type rb_block_type = u32; +#[repr(C)] +pub struct rb_block { + pub as_: rb_block__bindgen_ty_1, + pub type_: rb_block_type, +} +#[repr(C)] +pub struct rb_block__bindgen_ty_1 { + pub captured: __BindgenUnionField<rb_captured_block>, + pub symbol: __BindgenUnionField<VALUE>, + pub proc_: __BindgenUnionField<VALUE>, + pub bindgen_union_field: [u64; 3usize], +} pub type rb_control_frame_t = rb_control_frame_struct; +#[repr(C)] +pub struct rb_proc_t { + pub block: rb_block, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub __bindgen_padding_0: [u8; 7usize], +} +impl rb_proc_t { + #[inline] + pub fn is_from_method(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_from_method(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_from_method_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_from_method_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_lambda(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_lambda(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_lambda_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_lambda_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_isolated(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_isolated(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_isolated_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_isolated_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + is_from_method: ::std::os::raw::c_uint, + is_lambda: ::std::os::raw::c_uint, + is_isolated: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let is_from_method: u32 = unsafe { ::std::mem::transmute(is_from_method) }; + is_from_method as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let is_lambda: u32 = unsafe { ::std::mem::transmute(is_lambda) }; + is_lambda as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let is_isolated: u32 = unsafe { ::std::mem::transmute(is_isolated) }; + is_isolated as u64 + }); + __bindgen_bitfield_unit + } +} pub const VM_CHECKMATCH_TYPE_WHEN: vm_check_match_type = 1; pub const VM_CHECKMATCH_TYPE_CASE: vm_check_match_type = 2; pub const VM_CHECKMATCH_TYPE_RESCUE: vm_check_match_type = 3; @@ -391,37 +1410,21 @@ pub const VM_FRAME_FLAG_LAMBDA: vm_frame_env_flags = 256; pub const VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM: vm_frame_env_flags = 512; pub const VM_FRAME_FLAG_CFRAME_KW: vm_frame_env_flags = 1024; pub const VM_FRAME_FLAG_PASSED: vm_frame_env_flags = 2048; -pub const VM_FRAME_FLAG_NS_SWITCH: vm_frame_env_flags = 4096; -pub const VM_FRAME_FLAG_LOAD_ISEQ: vm_frame_env_flags = 8192; +pub const VM_FRAME_FLAG_BOX_REQUIRE: vm_frame_env_flags = 4096; pub const VM_ENV_FLAG_LOCAL: vm_frame_env_flags = 2; pub const VM_ENV_FLAG_ESCAPED: vm_frame_env_flags = 4; pub const VM_ENV_FLAG_WB_REQUIRED: vm_frame_env_flags = 8; pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; pub type vm_frame_env_flags = u32; -pub type attr_index_t = u32; +pub type attr_index_t = u16; pub type shape_id_t = u32; -pub type redblack_id_t = u32; -pub type redblack_node_t = redblack_node; -#[repr(C)] -pub struct rb_shape { - pub edges: *mut rb_id_table, - pub edge_name: ID, - pub next_field_index: attr_index_t, - pub capacity: attr_index_t, - pub type_: u8, - pub heap_index: u8, - pub flags: u8, - pub parent_id: shape_id_t, - pub ancestor_index: *mut redblack_node_t, -} -pub type rb_shape_t = rb_shape; -#[repr(C)] -pub struct redblack_node { - pub key: ID, - pub value: *mut rb_shape_t, - pub l: redblack_id_t, - pub r: redblack_id_t, -} +pub const SHAPE_ID_HEAP_INDEX_MASK: shape_id_fl_type = 29360128; +pub const SHAPE_ID_FL_FROZEN: shape_id_fl_type = 33554432; +pub const SHAPE_ID_FL_HAS_OBJECT_ID: shape_id_fl_type = 67108864; +pub const SHAPE_ID_FL_TOO_COMPLEX: shape_id_fl_type = 134217728; +pub const SHAPE_ID_FL_NON_CANONICAL_MASK: shape_id_fl_type = 100663296; +pub const SHAPE_ID_FLAGS_MASK: shape_id_fl_type = 264241152; +pub type shape_id_fl_type = u32; #[repr(C)] pub struct rb_cvar_class_tbl_entry { pub index: u32, @@ -579,151 +1582,227 @@ pub const YARVINSN_opt_and: ruby_vminsn_type = 90; pub const YARVINSN_opt_or: ruby_vminsn_type = 91; pub const YARVINSN_opt_aref: ruby_vminsn_type = 92; pub const YARVINSN_opt_aset: ruby_vminsn_type = 93; -pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 94; -pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 95; -pub const YARVINSN_opt_length: ruby_vminsn_type = 96; -pub const YARVINSN_opt_size: ruby_vminsn_type = 97; -pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 98; -pub const YARVINSN_opt_succ: ruby_vminsn_type = 99; -pub const YARVINSN_opt_not: ruby_vminsn_type = 100; -pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 101; -pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 102; -pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 103; -pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 104; -pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 105; -pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 106; -pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 107; -pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 108; -pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 109; -pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 110; -pub const YARVINSN_trace_nop: ruby_vminsn_type = 111; -pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 112; -pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 113; -pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 114; -pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 115; -pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 116; -pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 117; -pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 118; -pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 119; -pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 120; -pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 121; -pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 122; -pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 123; -pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 124; -pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 125; -pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 126; -pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 127; -pub const YARVINSN_trace_putnil: ruby_vminsn_type = 128; -pub const YARVINSN_trace_putself: ruby_vminsn_type = 129; -pub const YARVINSN_trace_putobject: ruby_vminsn_type = 130; -pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 131; -pub const YARVINSN_trace_putstring: ruby_vminsn_type = 132; -pub const YARVINSN_trace_putchilledstring: ruby_vminsn_type = 133; -pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 134; -pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 135; -pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 136; -pub const YARVINSN_trace_intern: ruby_vminsn_type = 137; -pub const YARVINSN_trace_newarray: ruby_vminsn_type = 138; -pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 139; -pub const YARVINSN_trace_duparray: ruby_vminsn_type = 140; -pub const YARVINSN_trace_duphash: ruby_vminsn_type = 141; -pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 142; -pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 143; -pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 144; -pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 145; -pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 146; -pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 147; -pub const YARVINSN_trace_newhash: ruby_vminsn_type = 148; -pub const YARVINSN_trace_newrange: ruby_vminsn_type = 149; -pub const YARVINSN_trace_pop: ruby_vminsn_type = 150; -pub const YARVINSN_trace_dup: ruby_vminsn_type = 151; -pub const YARVINSN_trace_dupn: ruby_vminsn_type = 152; -pub const YARVINSN_trace_swap: ruby_vminsn_type = 153; -pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 154; -pub const YARVINSN_trace_topn: ruby_vminsn_type = 155; -pub const YARVINSN_trace_setn: ruby_vminsn_type = 156; -pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 157; -pub const YARVINSN_trace_defined: ruby_vminsn_type = 158; -pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 159; -pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 160; -pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 161; -pub const YARVINSN_trace_checktype: ruby_vminsn_type = 162; -pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 163; -pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 164; -pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 165; -pub const YARVINSN_trace_send: ruby_vminsn_type = 166; -pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 167; -pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 168; -pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 169; -pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 170; -pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 171; -pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 172; -pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 173; -pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 174; -pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 175; -pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 176; -pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 177; -pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 178; -pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 179; -pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 180; -pub const YARVINSN_trace_leave: ruby_vminsn_type = 181; -pub const YARVINSN_trace_throw: ruby_vminsn_type = 182; -pub const YARVINSN_trace_jump: ruby_vminsn_type = 183; -pub const YARVINSN_trace_branchif: ruby_vminsn_type = 184; -pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 185; -pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 186; -pub const YARVINSN_trace_once: ruby_vminsn_type = 187; -pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 188; -pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 189; -pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 190; -pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 191; -pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 192; -pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 193; -pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 194; -pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 195; -pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 196; -pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 197; -pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 198; -pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 199; -pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 200; -pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 201; -pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 202; -pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 203; -pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 204; -pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 205; -pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 206; -pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 207; -pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 208; -pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 209; -pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 210; -pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 211; -pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 212; -pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 213; -pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 214; -pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 215; -pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 216; -pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 217; -pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 218; -pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 219; -pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 220; -pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 221; +pub const YARVINSN_opt_length: ruby_vminsn_type = 94; +pub const YARVINSN_opt_size: ruby_vminsn_type = 95; +pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 96; +pub const YARVINSN_opt_succ: ruby_vminsn_type = 97; +pub const YARVINSN_opt_not: ruby_vminsn_type = 98; +pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 99; +pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 100; +pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 101; +pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 102; +pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 103; +pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 104; +pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 105; +pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 106; +pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 107; +pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 108; +pub const YARVINSN_trace_nop: ruby_vminsn_type = 109; +pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 110; +pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 111; +pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 112; +pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 113; +pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 114; +pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 115; +pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 116; +pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 117; +pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 118; +pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 119; +pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 120; +pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 121; +pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 122; +pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 123; +pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 124; +pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 125; +pub const YARVINSN_trace_putnil: ruby_vminsn_type = 126; +pub const YARVINSN_trace_putself: ruby_vminsn_type = 127; +pub const YARVINSN_trace_putobject: ruby_vminsn_type = 128; +pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 129; +pub const YARVINSN_trace_putstring: ruby_vminsn_type = 130; +pub const YARVINSN_trace_putchilledstring: ruby_vminsn_type = 131; +pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 132; +pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 133; +pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 134; +pub const YARVINSN_trace_intern: ruby_vminsn_type = 135; +pub const YARVINSN_trace_newarray: ruby_vminsn_type = 136; +pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 137; +pub const YARVINSN_trace_duparray: ruby_vminsn_type = 138; +pub const YARVINSN_trace_duphash: ruby_vminsn_type = 139; +pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 140; +pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 141; +pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 142; +pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 143; +pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 144; +pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 145; +pub const YARVINSN_trace_newhash: ruby_vminsn_type = 146; +pub const YARVINSN_trace_newrange: ruby_vminsn_type = 147; +pub const YARVINSN_trace_pop: ruby_vminsn_type = 148; +pub const YARVINSN_trace_dup: ruby_vminsn_type = 149; +pub const YARVINSN_trace_dupn: ruby_vminsn_type = 150; +pub const YARVINSN_trace_swap: ruby_vminsn_type = 151; +pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 152; +pub const YARVINSN_trace_topn: ruby_vminsn_type = 153; +pub const YARVINSN_trace_setn: ruby_vminsn_type = 154; +pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 155; +pub const YARVINSN_trace_defined: ruby_vminsn_type = 156; +pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 157; +pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 158; +pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 159; +pub const YARVINSN_trace_checktype: ruby_vminsn_type = 160; +pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 161; +pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 162; +pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 163; +pub const YARVINSN_trace_send: ruby_vminsn_type = 164; +pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 165; +pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 166; +pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 167; +pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 168; +pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 169; +pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 170; +pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 171; +pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 172; +pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 173; +pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 174; +pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 175; +pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 176; +pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 177; +pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 178; +pub const YARVINSN_trace_leave: ruby_vminsn_type = 179; +pub const YARVINSN_trace_throw: ruby_vminsn_type = 180; +pub const YARVINSN_trace_jump: ruby_vminsn_type = 181; +pub const YARVINSN_trace_branchif: ruby_vminsn_type = 182; +pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 183; +pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 184; +pub const YARVINSN_trace_once: ruby_vminsn_type = 185; +pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 186; +pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 187; +pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 188; +pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 189; +pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 190; +pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 191; +pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 192; +pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 193; +pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 194; +pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 195; +pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 196; +pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 197; +pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 198; +pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 199; +pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 200; +pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 201; +pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 202; +pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 203; +pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 204; +pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 205; +pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 206; +pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 207; +pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 208; +pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 209; +pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 210; +pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 211; +pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 212; +pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 213; +pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 214; +pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 215; +pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 216; +pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 217; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 218; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 219; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 220; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 221; pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 222; -pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 223; -pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 224; -pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 225; -pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 226; -pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 227; -pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 228; -pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 229; -pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 230; -pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 231; -pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 232; -pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 233; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 223; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 224; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 248; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 249; pub type ruby_vminsn_type = u32; pub type rb_iseq_callback = ::std::option::Option< unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), >; +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data { + pub _bindgen_opaque_blob: [u64; 24usize], +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union iseq_compile_data__bindgen_ty_1 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_2 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_3 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_insn_info_entry { + pub line_no: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub events: rb_event_flag_t, +} +pub const CATCH_TYPE_RESCUE: rb_catch_type = 3; +pub const CATCH_TYPE_ENSURE: rb_catch_type = 5; +pub const CATCH_TYPE_RETRY: rb_catch_type = 7; +pub const CATCH_TYPE_BREAK: rb_catch_type = 9; +pub const CATCH_TYPE_REDO: rb_catch_type = 11; +pub const CATCH_TYPE_NEXT: rb_catch_type = 13; +pub type rb_catch_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_catch_table_entry { + pub type_: rb_catch_type, + pub iseq: *mut rb_iseq_t, + pub start: ::std::os::raw::c_uint, + pub end: ::std::os::raw::c_uint, + pub cont: ::std::os::raw::c_uint, + pub sp: ::std::os::raw::c_uint, +} +#[repr(C, packed)] +pub struct iseq_catch_table { + pub size: ::std::os::raw::c_uint, + pub entries: __IncompleteArrayField<iseq_catch_table_entry>, +} +#[repr(C)] +#[derive(Debug)] +pub struct iseq_compile_data_storage { + pub next: *mut iseq_compile_data_storage, + pub pos: ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub buff: __IncompleteArrayField<::std::os::raw::c_char>, +} pub const DEFINED_NOT_DEFINED: defined_type = 0; pub const DEFINED_NIL: defined_type = 1; pub const DEFINED_IVAR: defined_type = 2; @@ -743,7 +1822,25 @@ pub const DEFINED_REF: defined_type = 15; pub const DEFINED_FUNC: defined_type = 16; pub const DEFINED_CONST_FROM: defined_type = 17; pub type defined_type = u32; -pub type rb_iseq_param_keyword_struct = rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword; +pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; +pub type zjit_struct_offsets = u32; +pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; +pub const ROBJECT_OFFSET_AS_ARY: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_RSTRING_LEN: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_EC_CFP: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_INTERRUPT_MASK: jit_bindgen_constants = 36; +pub const RUBY_OFFSET_EC_THREAD_PTR: jit_bindgen_constants = 48; +pub const RUBY_OFFSET_EC_RACTOR_ID: jit_bindgen_constants = 64; +pub type jit_bindgen_constants = u32; +pub const rb_invalid_shape_id: shape_id_t = 4294967295; +pub type rb_iseq_param_keyword_struct = + rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct succ_index_table { + pub _address: u8, +} unsafe extern "C" { pub fn ruby_xfree(ptr: *mut ::std::os::raw::c_void); pub fn rb_class_attached_object(klass: VALUE) -> VALUE; @@ -755,9 +1852,10 @@ unsafe extern "C" { pub fn rb_gc_mark(obj: VALUE); pub fn rb_gc_mark_movable(obj: VALUE); pub fn rb_gc_location(obj: VALUE) -> VALUE; + pub fn rb_gc_enable() -> VALUE; + pub fn rb_gc_disable() -> VALUE; pub fn rb_gc_writebarrier(old: VALUE, young: VALUE); pub fn rb_class_get_superclass(klass: VALUE) -> VALUE; - pub static mut rb_cObject: VALUE; pub fn rb_funcallv( recv: VALUE, mid: ID, @@ -766,6 +1864,7 @@ unsafe extern "C" { ) -> VALUE; pub static mut rb_mKernel: VALUE; pub static mut rb_cBasicObject: VALUE; + pub static mut rb_cObject: VALUE; pub static mut rb_cArray: VALUE; pub static mut rb_cClass: VALUE; pub static mut rb_cFalseClass: VALUE; @@ -776,6 +1875,9 @@ unsafe extern "C" { pub static mut rb_cModule: VALUE; pub static mut rb_cNilClass: VALUE; pub static mut rb_cNumeric: VALUE; + pub static mut rb_cRange: VALUE; + pub static mut rb_cRegexp: VALUE; + pub static mut rb_cSet: VALUE; pub static mut rb_cString: VALUE; pub static mut rb_cSymbol: VALUE; pub static mut rb_cThread: VALUE; @@ -798,7 +1900,10 @@ unsafe extern "C" { pub fn rb_ary_resurrect(ary: VALUE) -> VALUE; pub fn rb_ary_cat(ary: VALUE, train: *const VALUE, len: ::std::os::raw::c_long) -> VALUE; pub fn rb_ary_push(ary: VALUE, elem: VALUE) -> VALUE; + pub fn rb_ary_pop(ary: VALUE) -> VALUE; + pub fn rb_ary_entry(ary: VALUE, off: ::std::os::raw::c_long) -> VALUE; pub fn rb_ary_clear(ary: VALUE) -> VALUE; + pub fn rb_ary_concat(lhs: VALUE, rhs: VALUE) -> VALUE; pub fn rb_hash_new() -> VALUE; pub fn rb_hash_aref(hash: VALUE, key: VALUE) -> VALUE; pub fn rb_hash_aset(hash: VALUE, key: VALUE, val: VALUE) -> VALUE; @@ -814,9 +1919,17 @@ unsafe extern "C" { pub fn rb_intern(name: *const ::std::os::raw::c_char) -> ID; pub fn rb_intern2(name: *const ::std::os::raw::c_char, len: ::std::os::raw::c_long) -> ID; pub fn rb_id2str(id: ID) -> VALUE; + pub fn rb_sym2str(symbol: VALUE) -> VALUE; pub fn rb_class2name(klass: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_class_new_instance_pass_kw( + argc: ::std::os::raw::c_int, + argv: *const VALUE, + klass: VALUE, + ) -> VALUE; pub fn rb_obj_is_kind_of(obj: VALUE, klass: VALUE) -> VALUE; + pub fn rb_obj_alloc(klass: VALUE) -> VALUE; pub fn rb_obj_frozen_p(obj: VALUE) -> VALUE; + pub fn rb_class_real(klass: VALUE) -> VALUE; pub fn rb_class_inherited_p(scion: VALUE, ascendant: VALUE) -> VALUE; pub fn rb_backref_get() -> VALUE; pub fn rb_range_new(beg: VALUE, end: VALUE, excl: ::std::os::raw::c_int) -> VALUE; @@ -834,9 +1947,9 @@ unsafe extern "C" { pub fn rb_str_intern(str_: VALUE) -> VALUE; pub fn rb_mod_name(mod_: VALUE) -> VALUE; pub fn rb_ivar_get(obj: VALUE, name: ID) -> VALUE; + pub fn rb_ivar_set(obj: VALUE, name: ID, val: VALUE) -> VALUE; pub fn rb_ivar_defined(obj: VALUE, name: ID) -> VALUE; pub fn rb_attr_get(obj: VALUE, name: ID) -> VALUE; - pub fn rb_obj_info_dump(obj: VALUE); pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE; pub fn rb_reg_new_ary(ary: VALUE, options: ::std::os::raw::c_int) -> VALUE; @@ -851,6 +1964,7 @@ unsafe extern "C" { elts: *const VALUE, ) -> VALUE; pub fn rb_vm_top_self() -> VALUE; + pub static mut rb_vm_insn_count: u64; pub fn rb_method_entry_at(obj: VALUE, id: ID) -> *const rb_method_entry_t; pub fn rb_callable_method_entry(klass: VALUE, id: ID) -> *const rb_callable_method_entry_t; pub fn rb_callable_method_entry_or_negative( @@ -868,17 +1982,25 @@ unsafe extern "C" { cfp: *const rb_control_frame_t, ) -> *const rb_callable_method_entry_t; pub fn rb_obj_info(obj: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_raw_obj_info( + buff: *mut ::std::os::raw::c_char, + buff_size: usize, + obj: VALUE, + ) -> *const ::std::os::raw::c_char; pub fn rb_ec_stack_check(ec: *mut rb_execution_context_struct) -> ::std::os::raw::c_int; + pub fn rb_gc_writebarrier_remember(obj: VALUE); pub fn rb_shape_id_offset() -> i32; - pub fn rb_shape_lookup(shape_id: shape_id_t) -> *mut rb_shape_t; pub fn rb_obj_shape_id(obj: VALUE) -> shape_id_t; - pub fn rb_shape_get_iv_index(shape: *mut rb_shape_t, id: ID, value: *mut attr_index_t) -> bool; - pub fn rb_shape_obj_too_complex_p(obj: VALUE) -> bool; - pub fn rb_shape_transition_add_ivar_no_warnings(obj: VALUE, id: ID) -> shape_id_t; - pub fn rb_shape_id(shape: *mut rb_shape_t) -> shape_id_t; + pub fn rb_shape_get_iv_index(shape_id: shape_id_t, id: ID, value: *mut attr_index_t) -> bool; + pub fn rb_shape_transition_add_ivar_no_warnings( + klass: VALUE, + original_shape_id: shape_id_t, + id: ID, + ) -> shape_id_t; + pub fn rb_ivar_get_at_no_ractor_check(obj: VALUE, index: attr_index_t) -> VALUE; pub fn rb_gvar_get(arg1: ID) -> VALUE; pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; - pub fn rb_ensure_iv_list_size(obj: VALUE, len: u32, newsize: u32); + pub fn rb_ensure_iv_list_size(obj: VALUE, current_len: u32, newsize: u32); pub fn rb_vm_barrier(); pub fn rb_str_byte_substr(str_: VALUE, beg: VALUE, len: VALUE) -> VALUE; pub fn rb_str_substr_two_fixnums( @@ -918,6 +2040,7 @@ unsafe extern "C" { pub fn rb_iseq_line_no(iseq: *const rb_iseq_t, pos: usize) -> ::std::os::raw::c_uint; pub fn rb_iseqw_to_iseq(iseqw: VALUE) -> *const rb_iseq_t; pub fn rb_iseq_label(iseq: *const rb_iseq_t) -> VALUE; + pub fn rb_iseq_defined_string(type_: defined_type) -> VALUE; pub fn rb_profile_frames( start: ::std::os::raw::c_int, limit: ::std::os::raw::c_int, @@ -925,32 +2048,41 @@ unsafe extern "C" { lines: *mut ::std::os::raw::c_int, ) -> ::std::os::raw::c_int; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); - pub fn rb_zjit_get_page_size() -> u32; - pub fn rb_zjit_reserve_addr_space(mem_size: u32) -> *mut u8; + pub fn rb_zjit_exit_locations_dict( + zjit_raw_samples: *mut VALUE, + zjit_line_samples: *mut ::std::os::raw::c_int, + samples_len: ::std::os::raw::c_int, + ) -> VALUE; pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; - pub fn rb_zjit_multi_ractor_p() -> bool; pub fn rb_zjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; - pub fn rb_zjit_vm_unlock( - recursive_lock_level: *mut ::std::os::raw::c_uint, - file: *const ::std::os::raw::c_char, - line: ::std::os::raw::c_int, - ); - pub fn rb_zjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); - pub fn rb_zjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_icache_invalidate( - start: *mut ::std::os::raw::c_void, - end: *mut ::std::os::raw::c_void, - ); - pub fn rb_zjit_vm_lock_then_barrier( - recursive_lock_level: *mut ::std::os::raw::c_uint, - file: *const ::std::os::raw::c_char, - line: ::std::os::raw::c_int, + pub fn rb_zjit_iseq_insn_set( + iseq: *const rb_iseq_t, + insn_idx: ::std::os::raw::c_uint, + bare_insn: ruby_vminsn_type, ); pub fn rb_iseq_get_zjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; pub fn rb_iseq_set_zjit_payload(iseq: *const rb_iseq_t, payload: *mut ::std::os::raw::c_void); pub fn rb_zjit_print_exception(); + pub fn rb_zjit_singleton_class_p(klass: VALUE) -> bool; + pub fn rb_zjit_defined_ivar(obj: VALUE, id: ID, pushval: VALUE) -> VALUE; + pub fn rb_zjit_method_tracing_currently_enabled() -> bool; + pub fn rb_zjit_insn_leaf(insn: ::std::os::raw::c_int, opes: *const VALUE) -> bool; + pub fn rb_zjit_local_id(iseq: *const rb_iseq_t, idx: ::std::os::raw::c_uint) -> ID; + pub fn rb_zjit_cme_is_cfunc( + me: *const rb_callable_method_entry_t, + func: *const ::std::os::raw::c_void, + ) -> bool; + pub fn rb_zjit_vm_search_method( + cd_owner: VALUE, + cd: *mut rb_call_data, + recv: VALUE, + ) -> *const rb_callable_method_entry_struct; + pub fn rb_zjit_class_initialized_p(klass: VALUE) -> bool; + pub fn rb_zjit_class_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; + pub fn rb_zjit_class_has_default_allocator(klass: VALUE) -> bool; + pub fn rb_vm_get_untagged_block_handler(reg_cfp: *mut rb_control_frame_t) -> VALUE; + pub fn rb_zjit_writebarrier_check_immediate(recv: VALUE, val: VALUE); pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE; pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int; @@ -980,10 +2112,22 @@ unsafe extern "C" { ) -> *mut rb_method_cfunc_t; pub fn rb_get_def_method_serial(def: *const rb_method_definition_t) -> usize; pub fn rb_get_def_original_id(def: *const rb_method_definition_t) -> ID; + pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; + pub fn rb_jit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t; + pub fn rb_optimized_call( + recv: *mut VALUE, + ec: *mut rb_execution_context_t, + argc: ::std::os::raw::c_int, + argv: *mut VALUE, + kw_splat: ::std::os::raw::c_int, + block_handler: VALUE, + ) -> VALUE; + pub fn rb_jit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_get_mct_argc(mct: *const rb_method_cfunc_t) -> ::std::os::raw::c_int; pub fn rb_get_mct_func(mct: *const rb_method_cfunc_t) -> *mut ::std::os::raw::c_void; pub fn rb_get_def_iseq_ptr(def: *mut rb_method_definition_t) -> *const rb_iseq_t; pub fn rb_get_iseq_body_local_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; + pub fn rb_get_iseq_body_parent_iseq(iseq: *const rb_iseq_t) -> *const rb_iseq_t; pub fn rb_get_iseq_body_local_table_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_get_iseq_body_iseq_encoded(iseq: *const rb_iseq_t) -> *mut VALUE; pub fn rb_get_iseq_body_stack_max(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; @@ -1018,11 +2162,42 @@ unsafe extern "C" { pub fn rb_FL_TEST(obj: VALUE, flags: VALUE) -> VALUE; pub fn rb_FL_TEST_RAW(obj: VALUE, flags: VALUE) -> VALUE; pub fn rb_RB_TYPE_P(obj: VALUE, t: ruby_value_type) -> bool; - pub fn rb_RSTRUCT_LEN(st: VALUE) -> ::std::os::raw::c_long; pub fn rb_get_call_data_ci(cd: *const rb_call_data) -> *const rb_callinfo; pub fn rb_BASIC_OP_UNREDEFINED_P(bop: ruby_basic_operators, klass: u32) -> bool; pub fn rb_RCLASS_ORIGIN(c: VALUE) -> VALUE; pub fn rb_assert_iseq_handle(handle: VALUE); + pub fn rb_assert_holding_vm_lock(); pub fn rb_IMEMO_TYPE_P(imemo: VALUE, imemo_type: imemo_type) -> ::std::os::raw::c_int; pub fn rb_assert_cme_handle(handle: VALUE); + pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; + pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; + pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); + pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); + pub fn rb_jit_shape_too_complex_p(shape_id: shape_id_t) -> bool; + pub fn rb_jit_multi_ractor_p() -> bool; + pub fn rb_jit_vm_lock_then_barrier( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_jit_vm_unlock( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; + pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); + pub fn rb_jit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE; + pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE; + pub fn rb_jit_str_concat_codepoint(str_: VALUE, codepoint: VALUE); + pub fn rb_jit_shape_capacity(shape_id: shape_id_t) -> attr_index_t; } diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index 26ad349e29..357c8b0c12 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -12,9 +12,11 @@ use crate::cruby::*; use std::collections::HashMap; use std::ffi::c_void; use crate::hir_type::{types, Type}; +use crate::hir; pub struct Annotations { cfuncs: HashMap<*mut c_void, FnProperties>, + builtin_funcs: HashMap<*mut c_void, FnProperties>, } /// Runtime behaviors of C functions that implement a Ruby method @@ -28,6 +30,20 @@ pub struct FnProperties { pub return_type: Type, /// Whether it's legal to remove the call if the result is unused pub elidable: bool, + pub inline: fn(&mut hir::Function, hir::BlockId, hir::InsnId, &[hir::InsnId], hir::InsnId) -> Option<hir::InsnId>, +} + +/// A safe default for un-annotated Ruby methods: we can't optimize them or their returned values. +impl Default for FnProperties { + fn default() -> Self { + Self { + no_gc: false, + leaf: false, + return_type: types::BasicObject, + elidable: false, + inline: no_inline, + } + } } impl Annotations { @@ -41,6 +57,12 @@ impl Annotations { }; self.cfuncs.get(&fn_ptr).copied() } + + /// Query about properties of a builtin function by its pointer + pub fn get_builtin_properties(&self, bf: *const rb_builtin_function) -> Option<FnProperties> { + let func_ptr = unsafe { (*bf).func_ptr as *mut c_void }; + self.builtin_funcs.get(&func_ptr).copied() + } } fn annotate_c_method(props_map: &mut HashMap<*mut c_void, FnProperties>, class: VALUE, method_name: &'static str, props: FnProperties) { @@ -59,27 +81,805 @@ fn annotate_c_method(props_map: &mut HashMap<*mut c_void, FnProperties>, class: props_map.insert(fn_ptr, props); } +/// Look up a method and find its builtin function pointer by parsing its ISEQ +/// We currently only support methods with exactly one invokebuiltin instruction +fn annotate_builtin_method(props_map: &mut HashMap<*mut c_void, FnProperties>, class: VALUE, method_name: &'static str, props: FnProperties) { + unsafe { + let method_id = rb_intern2(method_name.as_ptr().cast(), method_name.len().try_into().unwrap()); + let method = rb_method_entry_at(class, method_id); + if method.is_null() { + panic!("Method {}#{} not found", std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), method_name); + } + + // Cast ME to CME - they have identical layout + let cme = method.cast::<rb_callable_method_entry_t>(); + let def_type = get_cme_def_type(cme); + + if def_type != VM_METHOD_TYPE_ISEQ { + panic!("Method {}#{} is not an ISEQ method (type: {})", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name, def_type); + } + + // Get the ISEQ from the method definition + let iseq = get_def_iseq_ptr((*cme).def); + if iseq.is_null() { + panic!("Failed to get ISEQ for {}#{}", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name); + } + + // Get the size of the ISEQ in instruction units + let encoded_size = rb_iseq_encoded_size(iseq); + + // Scan through the ISEQ to find invokebuiltin instructions + let mut insn_idx: u32 = 0; + let mut func_ptr = std::ptr::null_mut::<c_void>(); + + while insn_idx < encoded_size { + // Get the PC for this instruction index + let pc = rb_iseq_pc_at_idx(iseq, insn_idx); + + // Get the opcode using the proper decoder + let opcode = rb_iseq_opcode_at_pc(iseq, pc); + + if opcode == YARVINSN_invokebuiltin as i32 || + opcode == YARVINSN_opt_invokebuiltin_delegate as i32 || + opcode == YARVINSN_opt_invokebuiltin_delegate_leave as i32 { + // The first operand is the builtin function pointer + let bf_value = *pc.add(1); + let bf_ptr: *const rb_builtin_function = bf_value.as_ptr(); + + if func_ptr.is_null() { + func_ptr = (*bf_ptr).func_ptr as *mut c_void; + } else { + panic!("Multiple invokebuiltin instructions found in ISEQ for {}#{}", + std::ffi::CStr::from_ptr(rb_class2name(class)).to_str().unwrap_or("?"), + method_name); + } + } + + // Move to the next instruction using the proper length + insn_idx = insn_idx.saturating_add(rb_insn_len(VALUE(opcode as usize)).try_into().unwrap()); + } + + // Only insert the properties if its iseq has exactly one invokebuiltin instruction + props_map.insert(func_ptr, props); + } +} + /// Gather annotations. Run this right after boot since the annotations /// are about the stock versions of methods. pub fn init() -> Annotations { let cfuncs = &mut HashMap::new(); + let builtin_funcs = &mut HashMap::new(); macro_rules! annotate { - ($module:ident, $method_name:literal, $return_type:expr, $($properties:ident),+) => { - let mut props = FnProperties { no_gc: false, leaf: false, elidable: false, return_type: $return_type }; + ($module:ident, $method_name:literal, $inline:ident) => { + let mut props = FnProperties::default(); + props.inline = $inline; + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; + $( + props.$properties = true; + )* + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; $( props.$properties = true; - )+ + )* + #[allow(unused_unsafe)] annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); } } - annotate!(rb_mKernel, "itself", types::BasicObject, no_gc, leaf, elidable); - annotate!(rb_cString, "bytesize", types::Fixnum, no_gc, leaf); - annotate!(rb_cModule, "name", types::StringExact.union(types::NilClassExact), no_gc, leaf, elidable); - annotate!(rb_cModule, "===", types::BoolExact, no_gc, leaf); + macro_rules! annotate_builtin { + ($module:ident, $method_name:literal, $return_type:expr) => { + annotate_builtin!($module, $method_name, $return_type, no_gc, leaf, elidable) + }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + $(props.$properties = true;)+ + annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; + $(props.$properties = true;)+ + annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); + } + } + + annotate!(rb_mKernel, "itself", inline_kernel_itself); + annotate!(rb_mKernel, "block_given?", inline_kernel_block_given_p); + annotate!(rb_mKernel, "===", inline_eqq); + annotate!(rb_mKernel, "is_a?", inline_kernel_is_a_p); + annotate!(rb_cString, "bytesize", inline_string_bytesize); + annotate!(rb_cString, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "length", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "getbyte", inline_string_getbyte); + annotate!(rb_cString, "setbyte", inline_string_setbyte); + annotate!(rb_cString, "empty?", inline_string_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cString, "<<", inline_string_append); + annotate!(rb_cString, "==", inline_string_eq); + // Not elidable; has a side effect of setting the encoding if ENC_CODERANGE_UNKNOWN. + // TOOD(max): Turn this into a load/compare. Will need to side-exit or do the full call if + // ENC_CODERANGE_UNKNOWN. + annotate!(rb_cString, "ascii_only?", types::BoolExact, no_gc, leaf); + annotate!(rb_cModule, "name", types::StringExact.union(types::NilClass), no_gc, leaf, elidable); + annotate!(rb_cModule, "===", inline_module_eqq, types::BoolExact, no_gc, leaf); + annotate!(rb_cArray, "length", inline_array_length, types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cArray, "empty?", inline_array_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cArray, "reverse", types::ArrayExact, leaf, elidable); + annotate!(rb_cArray, "join", types::StringExact); + annotate!(rb_cArray, "[]", inline_array_aref); + annotate!(rb_cArray, "[]=", inline_array_aset); + annotate!(rb_cArray, "<<", inline_array_push); + annotate!(rb_cArray, "push", inline_array_push); + annotate!(rb_cArray, "pop", inline_array_pop); + annotate!(rb_cHash, "[]", inline_hash_aref); + annotate!(rb_cHash, "[]=", inline_hash_aset); + annotate!(rb_cHash, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cHash, "empty?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cNilClass, "nil?", inline_nilclass_nil_p); + annotate!(rb_mKernel, "nil?", inline_kernel_nil_p); + annotate!(rb_mKernel, "respond_to?", inline_kernel_respond_to_p); + annotate!(rb_cBasicObject, "==", inline_basic_object_eq, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!", inline_basic_object_not, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!=", inline_basic_object_neq, types::BoolExact); + annotate!(rb_cBasicObject, "initialize", inline_basic_object_initialize); + annotate!(rb_cInteger, "succ", inline_integer_succ); + annotate!(rb_cInteger, "^", inline_integer_xor); + annotate!(rb_cInteger, "==", inline_integer_eq); + annotate!(rb_cInteger, "+", inline_integer_plus); + annotate!(rb_cInteger, "-", inline_integer_minus); + annotate!(rb_cInteger, "*", inline_integer_mult); + annotate!(rb_cInteger, "/", inline_integer_div); + annotate!(rb_cInteger, "%", inline_integer_mod); + annotate!(rb_cInteger, "&", inline_integer_and); + annotate!(rb_cInteger, "|", inline_integer_or); + annotate!(rb_cInteger, ">", inline_integer_gt); + annotate!(rb_cInteger, ">=", inline_integer_ge); + annotate!(rb_cInteger, "<", inline_integer_lt); + annotate!(rb_cInteger, "<=", inline_integer_le); + annotate!(rb_cInteger, "<<", inline_integer_lshift); + annotate!(rb_cInteger, ">>", inline_integer_rshift); + annotate!(rb_cInteger, "[]", inline_integer_aref); + annotate!(rb_cInteger, "to_s", types::StringExact); + annotate!(rb_cString, "to_s", inline_string_to_s, types::StringExact); + let thread_singleton = unsafe { rb_singleton_class(rb_cThread) }; + annotate!(thread_singleton, "current", inline_thread_current, types::BasicObject, no_gc, leaf); + + annotate_builtin!(rb_mKernel, "Float", types::Float); + annotate_builtin!(rb_mKernel, "Integer", types::Integer); + // TODO(max): Annotate rb_mKernel#class as returning types::Class. Right now there is a subtle + // type system bug that causes an issue if we make it return types::Class. + annotate_builtin!(rb_mKernel, "class", inline_kernel_class, types::HeapObject, leaf); + annotate_builtin!(rb_mKernel, "frozen?", types::BoolExact); + annotate_builtin!(rb_cSymbol, "name", types::StringExact); + annotate_builtin!(rb_cSymbol, "to_s", types::StringExact); Annotations { - cfuncs: std::mem::take(cfuncs) + cfuncs: std::mem::take(cfuncs), + builtin_funcs: std::mem::take(builtin_funcs), + } +} + +fn no_inline(_fun: &mut hir::Function, _block: hir::BlockId, _recv: hir::InsnId, _args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + None +} + +fn inline_string_to_s(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::StringExact, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + return Some(recv); + } + None +} + +fn inline_thread_current(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let ec = fun.push_insn(block, hir::Insn::LoadEC); + let thread_ptr = fun.push_insn(block, hir::Insn::LoadField { + recv: ec, + id: ID!(thread_ptr), + offset: RUBY_OFFSET_EC_THREAD_PTR as i32, + return_type: types::CPtr, + }); + let thread_self = fun.push_insn(block, hir::Insn::LoadField { + recv: thread_ptr, + id: ID!(self_), + offset: RUBY_OFFSET_THREAD_SELF as i32, + // TODO(max): Add Thread type. But Thread.current is not guaranteed to be an exact Thread. + // You can make subclasses... + return_type: types::BasicObject, + }); + Some(thread_self) +} + +fn inline_kernel_itself(_fun: &mut hir::Function, _block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() { + // No need to coerce the receiver; that is done by the SendWithoutBlock rewriting. + return Some(recv); + } + None +} + +fn inline_kernel_block_given_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + // TODO(max): In local iseq types that are not ISEQ_TYPE_METHOD, rewrite to Constant false. + Some(fun.push_insn(block, hir::Insn::IsBlockGiven)) +} + +fn inline_array_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index] = args { + if fun.likely_a(index, types::Fixnum, state) { + let index = fun.coerce_to(block, index, types::Fixnum, state); + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, state }); + let result = fun.push_insn(block, hir::Insn::ArrayAref { array: recv, index }); + return Some(result); + } + } + None +} + +fn inline_array_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index, val] = args { + if fun.likely_a(recv, types::ArrayExact, state) + && fun.likely_a(index, types::Fixnum, state) + { + let recv = fun.coerce_to(block, recv, types::ArrayExact, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let recv = fun.push_insn(block, hir::Insn::GuardNotFrozen { recv, state }); + let recv = fun.push_insn(block, hir::Insn::GuardNotShared { recv, state }); + + // Bounds check: unbox Fixnum index and guard 0 <= idx < length. + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, state }); + + let _ = fun.push_insn(block, hir::Insn::ArrayAset { array: recv, index, val }); + fun.push_insn(block, hir::Insn::WriteBarrier { recv, val }); + return Some(val); + } + } + None +} + +fn inline_array_push(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Inline only the case of `<<` or `push` when called with a single argument. + if let &[val] = args { + let _ = fun.push_insn(block, hir::Insn::ArrayPush { array: recv, val, state }); + return Some(recv); + } + None +} + +fn inline_array_pop(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Only inline the case of no arguments. + let &[] = args else { return None; }; + // We know that all Array are HeapObject, so no need to insert a GuardType(HeapObject). + let arr = fun.push_insn(block, hir::Insn::GuardNotFrozen { recv, state }); + Some(fun.push_insn(block, hir::Insn::ArrayPop { array: arr, state })) +} + +fn inline_hash_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let result = fun.push_insn(block, hir::Insn::HashAref { hash: recv, key, state }); + Some(result) + } else { + None + } +} + +fn inline_hash_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key, val] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let _ = fun.push_insn(block, hir::Insn::HashAset { hash: recv, key, val, state }); + // Hash#[]= returns the value, not the hash + Some(val) + } else { + None + } +} + +fn inline_string_bytesize(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + + let result = fun.push_insn(block, hir::Insn::BoxFixnum { + val: len, + state, + }); + + return Some(result); + } + None +} + +fn inline_string_getbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) { + // String#getbyte with a Fixnum is leaf and nogc; otherwise it may run arbitrary Ruby code + // when converting the index to a C integer. + let index = fun.coerce_to(block, index, types::Fixnum, state); + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + // TODO(max): Find a way to mark these guards as not needed for correctness... as in, once + // the data dependency is gone (say, the StringGetbyte is elided), they can also be elided. + // + // This is unlike most other guards. + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, state }); + let result = fun.push_insn(block, hir::Insn::StringGetbyte { string: recv, index: unboxed_index }); + return Some(result); + } + None +} + +fn inline_string_setbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index, value] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) && fun.likely_a(value, types::Fixnum, state) { + let index = fun.coerce_to(block, index, types::Fixnum, state); + let value = fun.coerce_to(block, value, types::Fixnum, state); + + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, state }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, state }); + // We know that all String are HeapObject, so no need to insert a GuardType(HeapObject). + let recv = fun.push_insn(block, hir::Insn::GuardNotFrozen { recv, state }); + let _ = fun.push_insn(block, hir::Insn::StringSetbyteFixnum { string: recv, index, value }); + // String#setbyte returns the fixnum provided as its `value` argument back to the caller. + Some(value) + } else { + None + } +} + +fn inline_string_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: ID!(len), + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let is_zero = fun.push_insn(block, hir::Insn::IsBitEqual { left: len, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: is_zero }); + Some(result) +} + +fn inline_string_append(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Inline only StringExact << String, which matches original type check from + // `vm_opt_ltlt`, which checks `RB_TYPE_P(obj, T_STRING)`. + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::String, state); + let _ = fun.push_insn(block, hir::Insn::StringAppend { recv, other, state }); + return Some(recv); + } + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::Fixnum, state); + let _ = fun.push_insn(block, hir::Insn::StringAppendCodepoint { recv, other, state }); + return Some(recv); + } + None +} + +fn inline_string_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.likely_a(recv, types::String, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let other = fun.coerce_to(block, other, types::String, state); + let return_type = types::BoolExact; + let elidable = true; + // TODO(max): Make StringEqual its own opcode so that we can later constant-fold StringEqual(a, a) => true + let result = fun.push_insn(block, hir::Insn::CCall { + cfunc: rb_yarv_str_eql_internal as *const u8, + recv, + args: vec![other], + name: ID!(string_eq), + return_type, + elidable, + }); + return Some(result); } + None +} + +fn inline_module_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(recv, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: other, class: recv }); + return Some(result); + } + None +} + +fn inline_array_length(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let result = fun.push_insn(block, hir::Insn::BoxFixnum { val: length_cint, state }); + return Some(result); + } + None +} + +fn inline_array_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let result_c = fun.push_insn(block, hir::Insn::IsBitEqual { left: length_cint, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: result_c }); + return Some(result); + } + None +} + +fn inline_integer_succ(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + if fun.likely_a(recv, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(VALUE::fixnum_from_usize(1)) }); + let result = fun.push_insn(block, hir::Insn::FixnumAdd { left, right, state }); + return Some(result); + } + None +} + +fn inline_integer_xor(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[right] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumXor { left, right }); + return Some(result); + } + None +} + +fn try_inline_fixnum_op(fun: &mut hir::Function, block: hir::BlockId, f: &dyn Fn(hir::InsnId, hir::InsnId) -> hir::Insn, bop: u32, left: hir::InsnId, right: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, INTEGER_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + return None; + } + if fun.likely_a(left, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + if bop == BOP_NEQ { + // For opt_neq, the interpreter checks that both neq and eq are unchanged. + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ }, state }); + } + // Rely on the MethodRedefined PatchPoint for other bops. + let left = fun.coerce_to(block, left, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + return Some(fun.push_insn(block, f(left, right))); + } + None +} + +fn inline_integer_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumEq { left, right }, BOP_EQ, recv, other, state) +} + +fn inline_integer_plus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAdd { left, right, state }, BOP_PLUS, recv, other, state) +} + +fn inline_integer_minus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumSub { left, right, state }, BOP_MINUS, recv, other, state) +} + +fn inline_integer_mult(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMult { left, right, state }, BOP_MULT, recv, other, state) +} + +fn inline_integer_div(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumDiv { left, right, state }, BOP_DIV, recv, other, state) +} + +fn inline_integer_mod(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMod { left, right, state }, BOP_MOD, recv, other, state) +} + +fn inline_integer_and(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAnd { left, right, }, BOP_AND, recv, other, state) +} + +fn inline_integer_or(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumOr { left, right, }, BOP_OR, recv, other, state) +} + +fn inline_integer_gt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGt { left, right }, BOP_GT, recv, other, state) +} + +fn inline_integer_ge(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGe { left, right }, BOP_GE, recv, other, state) +} + +fn inline_integer_lt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLt { left, right }, BOP_LT, recv, other, state) +} + +fn inline_integer_le(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLe { left, right }, BOP_LE, recv, other, state) +} + +fn inline_integer_lshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLShift { left, right, state }, BOP_LTLT, recv, other, state) +} + +fn inline_integer_rshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + // TODO(max): If other_value > 63, rewrite to constant zero. + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumRShift { left, right }, BOP_GTGT, recv, other, state) +} + +fn inline_integer_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(index, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::Fixnum, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumAref { recv, index }); + return Some(result); + } + None +} + +fn inline_basic_object_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_basic_object_not(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.type_of(recv).is_known_truthy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) }); + return Some(result); + } + if fun.type_of(recv).is_known_falsy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) }); + return Some(result); + } + None +} + +fn inline_basic_object_neq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let result = try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumNeq { left, right }, BOP_NEQ, recv, other, state); + if result.is_some() { + return result; + } + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitNotEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_basic_object_initialize(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qnil) }); + Some(result) +} + +fn inline_nilclass_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) })) +} + +fn inline_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_kernel_is_a_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(other, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: recv, class: other }); + return Some(result); + } + None +} + +fn inline_kernel_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) +} + +fn inline_kernel_respond_to_p( + fun: &mut hir::Function, + block: hir::BlockId, + recv: hir::InsnId, + args: &[hir::InsnId], + state: hir::InsnId, +) -> Option<hir::InsnId> { + // Parse arguments: respond_to?(method_name, allow_priv = false) + let (method_name, allow_priv) = match *args { + [method_name] => (method_name, false), + [method_name, arg] => match fun.type_of(arg) { + t if t.is_known_truthy() => (method_name, true), + t if t.is_known_falsy() => (method_name, false), + // Unknown type; bail out + _ => return None, + }, + // Unknown args; bail out + _ => return None, + }; + + // Method name must be a static symbol + let method_name = fun.type_of(method_name).ruby_object()?; + if !method_name.static_sym_p() { + return None; + } + + // The receiver must have a known class to call `respond_to?` on + // TODO: This is technically overly strict. This would also work if all of the + // observed objects at this point agree on `respond_to?` and we can add many patchpoints. + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + + // Get the method ID and its corresponding callable method entry + let mid = unsafe { rb_sym2id(method_name) }; + let target_cme = unsafe { rb_callable_method_entry_or_negative(recv_class, mid) }; + assert!( + !target_cme.is_null(), + "Should never be null, as in that case we will be returned a \"negative CME\"" + ); + + let cme_def_type = unsafe { get_cme_def_type(target_cme) }; + + // Cannot inline a refined method, since their refinement depends on lexical scope + if cme_def_type == VM_METHOD_TYPE_REFINED { + return None; + } + + let visibility = match cme_def_type { + VM_METHOD_TYPE_UNDEF => METHOD_VISI_UNDEF, + _ => unsafe { METHOD_ENTRY_VISI(target_cme) }, + }; + + let result = match (visibility, allow_priv) { + // Method undefined; check `respond_to_missing?` + (METHOD_VISI_UNDEF, _) => { + let respond_to_missing = ID!(respond_to_missing); + if unsafe { rb_method_basic_definition_p(recv_class, respond_to_missing) } == 0 { + return None; // Custom definition of respond_to_missing?, so cannot inline + } + let respond_to_missing_cme = + unsafe { rb_callable_method_entry(recv_class, respond_to_missing) }; + // Protect against redefinition of `respond_to_missing?` + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::NoTracePoint, + state, + }, + ); + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: respond_to_missing, + cme: respond_to_missing_cme, + }, + state, + }, + ); + Qfalse + } + // Private method with allow priv=false, so `respond_to?` returns false + (METHOD_VISI_PRIVATE, false) => Qfalse, + // Public method or allow_priv=true: check if implemented + (METHOD_VISI_PUBLIC, _) | (_, true) => { + if cme_def_type == VM_METHOD_TYPE_NOTIMPLEMENTED { + // C method with rb_f_notimplement(). `respond_to?` returns false + // without consulting `respond_to_missing?`. See also: rb_add_method_cfunc() + Qfalse + } else { + Qtrue + } + } + (_, _) => return None, // not public and include_all not known, can't compile + }; + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + return None; + } + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::NoTracePoint, state }); + fun.push_insn(block, hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: mid, + cme: target_cme + }, state + }); + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(result) })) +} + +fn inline_kernel_class(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + let real_class = unsafe { rb_class_real(recv_class) }; + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(real_class) })) } diff --git a/zjit/src/disasm.rs b/zjit/src/disasm.rs index 5c7a7be704..d4f6591594 100644 --- a/zjit/src/disasm.rs +++ b/zjit/src/disasm.rs @@ -1,8 +1,5 @@ use crate::asm::CodeBlock; -pub const BOLD_BEGIN: &str = "\x1b[1m"; -pub const BOLD_END: &str = "\x1b[22m"; - pub fn disasm_addr_range(cb: &CodeBlock, start_addr: usize, end_addr: usize) -> String { use std::fmt::Write; @@ -36,16 +33,20 @@ pub fn disasm_addr_range(cb: &CodeBlock, start_addr: usize, end_addr: usize) -> let start_addr = 0; let insns = cs.disasm_all(code_slice, start_addr as u64).unwrap(); + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + // For each instruction in this block for insn in insns.as_ref() { // Comments for this block if let Some(comment_list) = cb.comments_at(insn.address() as usize) { for comment in comment_list { - writeln!(&mut out, " {BOLD_BEGIN}# {comment}{BOLD_END}").unwrap(); + writeln!(&mut out, " {bold_begin}# {comment}{bold_end}").unwrap(); } } - writeln!(&mut out, " {insn}").unwrap(); + writeln!(&mut out, " {}", format!("{insn}").trim()).unwrap(); } - return out; + out } diff --git a/zjit/src/distribution.rs b/zjit/src/distribution.rs new file mode 100644 index 0000000000..2c6ffb3ae6 --- /dev/null +++ b/zjit/src/distribution.rs @@ -0,0 +1,276 @@ +//! Type frequency distribution tracker. + +/// This implementation was inspired by the type feedback module from Google's S6, which was +/// written in C++ for use with Python. This is a new implementation in Rust created for use with +/// Ruby instead of Python. +#[derive(Debug, Clone)] +pub struct Distribution<T: Copy + PartialEq + Default, const N: usize> { + /// buckets and counts have the same length + /// `buckets[0]` is always the most common item + buckets: [T; N], + counts: [usize; N], + /// if there is no more room, increment the fallback + other: usize, + // TODO(max): Add count disparity, which can help determine when to reset the distribution +} + +impl<T: Copy + PartialEq + Default, const N: usize> Distribution<T, N> { + pub fn new() -> Self { + Self { buckets: [Default::default(); N], counts: [0; N], other: 0 } + } + + pub fn observe(&mut self, item: T) { + for (bucket, count) in self.buckets.iter_mut().zip(self.counts.iter_mut()) { + if *bucket == item || *count == 0 { + *bucket = item; + *count += 1; + // Keep the most frequent item at the front + self.bubble_up(); + return; + } + } + self.other += 1; + } + + /// Keep the highest counted bucket at index 0 + fn bubble_up(&mut self) { + if N == 0 { return; } + let max_index = self.counts.into_iter().enumerate().max_by_key(|(_, val)| *val).unwrap().0; + if max_index != 0 { + self.counts.swap(0, max_index); + self.buckets.swap(0, max_index); + } + } + + pub fn each_item(&self) -> impl Iterator<Item = T> + '_ { + self.buckets.iter().zip(self.counts.iter()) + .filter_map(|(&bucket, &count)| if count > 0 { Some(bucket) } else { None }) + } + + pub fn each_item_mut(&mut self) -> impl Iterator<Item = &mut T> + '_ { + self.buckets.iter_mut().zip(self.counts.iter()) + .filter_map(|(bucket, &count)| if count > 0 { Some(bucket) } else { None }) + } +} + +#[derive(PartialEq, Debug, Clone, Copy)] +enum DistributionKind { + /// No types seen + Empty, + /// One type seen + Monomorphic, + /// Between 2 and (fixed) N types seen + Polymorphic, + /// Polymorphic, but with a significant skew towards one type + SkewedPolymorphic, + /// More than N types seen with no clear winner + Megamorphic, + /// Megamorphic, but with a significant skew towards one type + SkewedMegamorphic, +} + +#[derive(Debug)] +pub struct DistributionSummary<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> { + kind: DistributionKind, + buckets: [T; N], + // TODO(max): Determine if we need some notion of stability +} + +const SKEW_THRESHOLD: f64 = 0.75; + +impl<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> DistributionSummary<T, N> { + pub fn new(dist: &Distribution<T, N>) -> Self { + #[cfg(debug_assertions)] + { + let first_count = dist.counts[0]; + for &count in &dist.counts[1..] { + assert!(first_count >= count, "First count should be the largest"); + } + } + let num_seen = dist.counts.iter().sum::<usize>() + dist.other; + let kind = if dist.other == 0 { + // Seen <= N types total + if dist.counts[0] == 0 { + DistributionKind::Empty + } else if dist.counts[1] == 0 { + DistributionKind::Monomorphic + } else if (dist.counts[0] as f64)/(num_seen as f64) >= SKEW_THRESHOLD { + DistributionKind::SkewedPolymorphic + } else { + DistributionKind::Polymorphic + } + } else { + // Seen > N types total; considered megamorphic + if (dist.counts[0] as f64)/(num_seen as f64) >= SKEW_THRESHOLD { + DistributionKind::SkewedMegamorphic + } else { + DistributionKind::Megamorphic + } + }; + Self { kind, buckets: dist.buckets } + } + + pub fn is_monomorphic(&self) -> bool { + self.kind == DistributionKind::Monomorphic + } + + pub fn is_polymorphic(&self) -> bool { + self.kind == DistributionKind::Polymorphic + } + + pub fn is_skewed_polymorphic(&self) -> bool { + self.kind == DistributionKind::SkewedPolymorphic + } + + pub fn is_megamorphic(&self) -> bool { + self.kind == DistributionKind::Megamorphic + } + + pub fn is_skewed_megamorphic(&self) -> bool { + self.kind == DistributionKind::SkewedMegamorphic + } + + pub fn bucket(&self, idx: usize) -> T { + assert!(idx < N, "index {idx} out of bounds for buckets[{N}]"); + self.buckets[idx] + } +} + +#[cfg(test)] +mod distribution_tests { + use super::*; + + #[test] + fn start_empty() { + let dist = Distribution::<usize, 4>::new(); + assert_eq!(dist.other, 0); + assert!(dist.counts.iter().all(|&b| b == 0)); + } + + #[test] + fn observe_adds_record() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + assert_eq!(dist.buckets[0], 10); + assert_eq!(dist.counts[0], 1); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_increments_record() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + assert_eq!(dist.buckets[0], 10); + assert_eq!(dist.counts[0], 2); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_two() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(11); + assert_eq!(dist.buckets[0], 11); + assert_eq!(dist.counts[0], 3); + assert_eq!(dist.buckets[1], 10); + assert_eq!(dist.counts[1], 2); + assert_eq!(dist.other, 0); + } + + #[test] + fn observe_with_max_increments_other() { + let mut dist = Distribution::<usize, 0>::new(); + dist.observe(10); + assert!(dist.buckets.is_empty()); + assert!(dist.counts.is_empty()); + assert_eq!(dist.other, 1); + } + + #[test] + fn empty_distribution_returns_empty_summary() { + let dist = Distribution::<usize, 4>::new(); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Empty); + } + + #[test] + fn monomorphic_distribution_returns_monomorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(10); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Monomorphic); + assert_eq!(summary.buckets[0], 10); + } + + #[test] + fn polymorphic_distribution_returns_polymorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Polymorphic); + assert_eq!(summary.buckets[0], 11); + assert_eq!(summary.buckets[1], 10); + } + + #[test] + fn skewed_polymorphic_distribution_returns_skewed_polymorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::SkewedPolymorphic); + assert_eq!(summary.buckets[0], 11); + assert_eq!(summary.buckets[1], 10); + } + + #[test] + fn megamorphic_distribution_returns_megamorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(12); + dist.observe(13); + dist.observe(14); + dist.observe(11); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::Megamorphic); + assert_eq!(summary.buckets[0], 11); + } + + #[test] + fn skewed_megamorphic_distribution_returns_skewed_megamorphic_summary() { + let mut dist = Distribution::<usize, 4>::new(); + dist.observe(10); + dist.observe(11); + dist.observe(11); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(12); + dist.observe(13); + dist.observe(14); + let summary = DistributionSummary::new(&dist); + assert_eq!(summary.kind, DistributionKind::SkewedMegamorphic); + assert_eq!(summary.buckets[0], 12); + } +} diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs new file mode 100644 index 0000000000..40230ccc8d --- /dev/null +++ b/zjit/src/gc.rs @@ -0,0 +1,211 @@ +//! This module is responsible for marking/moving objects on GC. + +use std::ptr::null; +use std::{ffi::c_void, ops::Range}; +use crate::{cruby::*, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; +use crate::payload::{IseqPayload, IseqVersionRef, get_or_create_iseq_payload}; +use crate::stats::Counter::gc_time_ns; +use crate::state::gc_mark_raw_samples; + +/// GC callback for marking GC objects in the per-ISEQ payload. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_mark(payload: *mut c_void) { + let payload = if payload.is_null() { + return; // nothing to mark + } else { + // SAFETY: The GC takes the VM lock while marking, which + // we assert, so we should be synchronized and data race free. + // + // For aliasing, having the VM lock hopefully also implies that no one + // else has an overlapping &mut IseqPayload. + unsafe { + rb_assert_holding_vm_lock(); + &*(payload as *const IseqPayload) + } + }; + with_time_stat(gc_time_ns, || iseq_mark(payload)); +} + +/// GC callback for updating GC objects in the per-ISEQ payload. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_update_references(payload: *mut c_void) { + let payload = if payload.is_null() { + return; // nothing to update + } else { + // SAFETY: The GC takes the VM lock while marking, which + // we assert, so we should be synchronized and data race free. + // + // For aliasing, having the VM lock hopefully also implies that no one + // else has an overlapping &mut IseqPayload. + unsafe { + rb_assert_holding_vm_lock(); + &mut *(payload as *mut IseqPayload) + } + }; + with_time_stat(gc_time_ns, || iseq_update_references(payload)); +} + +/// GC callback for finalizing an ISEQ +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_free(iseq: IseqPtr) { + if !ZJITState::has_instance() { + return; + } + + // TODO(Shopify/ruby#682): Free `IseqPayload` + let payload = get_or_create_iseq_payload(iseq); + for version in payload.versions.iter_mut() { + unsafe { version.as_mut() }.iseq = null(); + } + + let invariants = ZJITState::get_invariants(); + invariants.forget_iseq(iseq); +} + +/// GC callback for finalizing a CME +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_cme_free(cme: *const rb_callable_method_entry_struct) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_cme(cme); +} + +/// GC callback for finalizing a class +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_klass_free(klass: VALUE) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_klass(klass); +} + +/// GC callback for updating object references after all object moves +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_update_references() { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.update_references(); +} + +fn iseq_mark(payload: &IseqPayload) { + // Mark objects retained by profiling instructions + payload.profile.each_object(|object| { + unsafe { rb_gc_mark_movable(object); } + }); + + // Mark objects baked in JIT code + let cb = ZJITState::get_code_block(); + for version in payload.versions.iter() { + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + // Creating an unaligned pointer is well defined unlike in C. + let value_ptr = value_ptr as *const VALUE; + + unsafe { + let object = value_ptr.read_unaligned(); + rb_gc_mark_movable(object); + } + } + } +} + +/// This is a mirror of [iseq_mark]. +fn iseq_update_references(payload: &mut IseqPayload) { + // Move objects retained by profiling instructions + payload.profile.each_object_mut(|old_object| { + let new_object = unsafe { rb_gc_location(*old_object) }; + if *old_object != new_object { + *old_object = new_object; + } + }); + + for &version in payload.versions.iter() { + iseq_version_update_references(version); + } +} + +fn iseq_version_update_references(mut version: IseqVersionRef) { + // Move ISEQ in the payload + unsafe { version.as_mut() }.iseq = unsafe { rb_gc_location(version.as_ref().iseq.into()) }.as_iseq(); + + // Move ISEQ references in incoming IseqCalls + for iseq_call in unsafe { version.as_mut() }.incoming.iter_mut() { + let old_iseq = iseq_call.iseq.get(); + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.iseq.set(new_iseq); + } + } + + // Move ISEQ references in outgoing IseqCalls + for iseq_call in unsafe { version.as_mut() }.outgoing.iter_mut() { + let old_iseq = iseq_call.iseq.get(); + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.iseq.set(new_iseq); + } + } + + // Move objects baked in JIT code + let cb = ZJITState::get_code_block(); + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + // Creating an unaligned pointer is well defined unlike in C. + let value_ptr = value_ptr as *const VALUE; + + let object = unsafe { value_ptr.read_unaligned() }; + let new_addr = unsafe { rb_gc_location(object) }; + + // Only write when the VALUE moves, to be copy-on-write friendly. + if new_addr != object { + for (byte_idx, &byte) in new_addr.as_u64().to_le_bytes().iter().enumerate() { + let byte_code_ptr = offset.add_bytes(byte_idx); + cb.write_mem(byte_code_ptr, byte).expect("patching existing code should be within bounds"); + } + } + } + cb.mark_all_executable(); +} + +/// Append a set of gc_offsets to the iseq's payload +pub fn append_gc_offsets(iseq: IseqPtr, mut version: IseqVersionRef, offsets: &Vec<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.extend(offsets); + + // Call writebarrier on each newly added value + let cb = ZJITState::get_code_block(); + for &offset in offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + let value_ptr = value_ptr as *const VALUE; + unsafe { + let object = value_ptr.read_unaligned(); + VALUE::from(iseq).write_barrier(object); + } + } +} + +/// Remove GC offsets that overlap with a given removed_range. +/// We do this when invalidation rewrites some code with a jump instruction +/// and GC offsets are corrupted by the rewrite, assuming no on-stack code +/// will step into the instruction with the GC offsets after invalidation. +pub fn remove_gc_offsets(mut version: IseqVersionRef, removed_range: &Range<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.retain(|&gc_offset| { + let offset_range = gc_offset..(gc_offset.add_bytes(SIZEOF_VALUE)); + !ranges_overlap(&offset_range, removed_range) + }); +} + +/// Return true if given `Range<CodePtr>` ranges overlap with each other +fn ranges_overlap<T>(left: &Range<T>, right: &Range<T>) -> bool where T: PartialOrd { + left.start < right.end && right.start < left.end +} + +/// Callback for marking GC objects inside [crate::invariants::Invariants]. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_mark() { + gc_mark_raw_samples(); +} diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 8cb7093ab8..6c2bd09ad3 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -3,22 +3,24 @@ // We use the YARV bytecode constants which have a CRuby-style name #![allow(non_upper_case_globals)] +#![allow(clippy::if_same_then_else)] +#![allow(clippy::match_like_matches_macro)] use crate::{ - cruby::*, - options::{get_option, DumpHIR}, - profile::{get_or_create_iseq_payload, IseqPayload}, - state::ZJITState, - cast::IntoUsize, + backend::lir::C_ARG_OPNDS, + cast::IntoUsize, codegen::local_idx_to_ep_offset, cruby::*, invariants::has_singleton_class_of, payload::{get_or_create_iseq_payload, IseqPayload}, options::{debug, get_option, DumpHIR}, state::ZJITState, json::Json }; use std::{ - cell::RefCell, - collections::{HashMap, HashSet, VecDeque}, - ffi::{c_int, c_void}, - mem::{align_of, size_of}, - ptr, - slice::Iter + cell::RefCell, collections::{BTreeSet, HashMap, HashSet, VecDeque}, ffi::{c_void, c_uint, c_int, CStr}, fmt::Display, mem::{align_of, size_of}, ptr, slice::Iter }; use crate::hir_type::{Type, types}; +use crate::hir_effect::{Effect, abstract_heaps, effects}; +use crate::bitset::BitSet; +use crate::profile::{TypeDistributionSummary, ProfiledType}; +use crate::stats::Counter; +use SendFallbackReason::*; + +mod tests; +mod opt_tests; /// An index of an [`Insn`] in a [`Function`]. This is a popular /// type since this effectively acts as a pointer to an [`Insn`]. @@ -26,9 +28,9 @@ use crate::hir_type::{Type, types}; #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Debug)] pub struct InsnId(pub usize); -impl Into<usize> for InsnId { - fn into(self) -> usize { - self.0 +impl From<InsnId> for usize { + fn from(val: InsnId) -> Self { + val.0 } } @@ -39,15 +41,24 @@ impl std::fmt::Display for InsnId { } /// The index of a [`Block`], which effectively acts like a pointer. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] pub struct BlockId(pub usize); +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 + } +} + impl std::fmt::Display for BlockId { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "bb{}", self.0) } } +type InsnSet = BitSet<InsnId>; +type BlockSet = BitSet<BlockId>; + fn write_vec<T: std::fmt::Display>(f: &mut std::fmt::Formatter, objs: &Vec<T>) -> std::fmt::Result { write!(f, "[")?; let mut prefix = ""; @@ -65,7 +76,7 @@ impl std::fmt::Display for VALUE { } impl VALUE { - pub fn print(self, ptr_map: &PtrPrintMap) -> VALUEPrinter { + pub fn print(self, ptr_map: &PtrPrintMap) -> VALUEPrinter<'_> { VALUEPrinter { inner: self, ptr_map } } } @@ -106,11 +117,6 @@ impl std::fmt::Display for BranchEdge { } } -#[derive(Debug, PartialEq, Clone)] -pub struct CallInfo { - pub method_name: String, -} - /// Invalidation reasons #[derive(Debug, Clone, Copy)] pub enum Invariant { @@ -126,22 +132,73 @@ pub enum Invariant { klass: VALUE, /// The method ID of the method we want to assume unchanged method: ID, + /// The callable method entry that we want to track + cme: *const rb_callable_method_entry_t, }, /// A list of constant expression path segments that must have not been written to for the /// following code to be valid. StableConstantNames { idlist: *const ID, }, + /// TracePoint is not enabled. If TracePoint is enabled, this is invalidated. + NoTracePoint, + /// cfp->ep is not escaped to the heap on the ISEQ + NoEPEscape(IseqPtr), /// There is one ractor running. If a non-root ractor gets spawned, this is invalidated. SingleRactorMode, + /// Objects of this class have no singleton class. + /// When a singleton class is created for an object of this class, this is invalidated. + NoSingletonClass { + klass: VALUE, + }, } impl Invariant { - pub fn print(self, ptr_map: &PtrPrintMap) -> InvariantPrinter { + pub fn print(self, ptr_map: &PtrPrintMap) -> InvariantPrinter<'_> { InvariantPrinter { inner: self, ptr_map } } } +impl Display for Invariant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpecialObjectType { + VMCore = 1, + CBase = 2, + ConstBase = 3, +} + +impl From<u32> for SpecialObjectType { + fn from(value: u32) -> Self { + match value { + VM_SPECIAL_OBJECT_VMCORE => SpecialObjectType::VMCore, + VM_SPECIAL_OBJECT_CBASE => SpecialObjectType::CBase, + VM_SPECIAL_OBJECT_CONST_BASE => SpecialObjectType::ConstBase, + _ => panic!("Invalid special object type: {value}"), + } + } +} + +impl From<SpecialObjectType> for u64 { + fn from(special_type: SpecialObjectType) -> Self { + special_type as u64 + } +} + +impl std::fmt::Display for SpecialObjectType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SpecialObjectType::VMCore => write!(f, "VMCore"), + SpecialObjectType::CBase => write!(f, "CBase"), + SpecialObjectType::ConstBase => write!(f, "ConstBase"), + } + } +} + /// Print adaptor for [`Invariant`]. See [`PtrPrintMap`]. pub struct InvariantPrinter<'a> { inner: Invariant, @@ -155,33 +212,58 @@ impl<'a> std::fmt::Display for InvariantPrinter<'a> { write!(f, "BOPRedefined(")?; match klass { INTEGER_REDEFINED_OP_FLAG => write!(f, "INTEGER_REDEFINED_OP_FLAG")?, + STRING_REDEFINED_OP_FLAG => write!(f, "STRING_REDEFINED_OP_FLAG")?, ARRAY_REDEFINED_OP_FLAG => write!(f, "ARRAY_REDEFINED_OP_FLAG")?, + HASH_REDEFINED_OP_FLAG => write!(f, "HASH_REDEFINED_OP_FLAG")?, _ => write!(f, "{klass}")?, } write!(f, ", ")?; match bop { - BOP_PLUS => write!(f, "BOP_PLUS")?, - BOP_MINUS => write!(f, "BOP_MINUS")?, - BOP_MULT => write!(f, "BOP_MULT")?, - BOP_DIV => write!(f, "BOP_DIV")?, - BOP_MOD => write!(f, "BOP_MOD")?, - BOP_EQ => write!(f, "BOP_EQ")?, - BOP_NEQ => write!(f, "BOP_NEQ")?, - BOP_LT => write!(f, "BOP_LT")?, - BOP_LE => write!(f, "BOP_LE")?, - BOP_GT => write!(f, "BOP_GT")?, - BOP_GE => write!(f, "BOP_GE")?, - BOP_MAX => write!(f, "BOP_MAX")?, + BOP_PLUS => write!(f, "BOP_PLUS")?, + BOP_MINUS => write!(f, "BOP_MINUS")?, + BOP_MULT => write!(f, "BOP_MULT")?, + BOP_DIV => write!(f, "BOP_DIV")?, + BOP_MOD => write!(f, "BOP_MOD")?, + BOP_EQ => write!(f, "BOP_EQ")?, + BOP_EQQ => write!(f, "BOP_EQQ")?, + BOP_LT => write!(f, "BOP_LT")?, + BOP_LE => write!(f, "BOP_LE")?, + BOP_LTLT => write!(f, "BOP_LTLT")?, + BOP_AREF => write!(f, "BOP_AREF")?, + BOP_ASET => write!(f, "BOP_ASET")?, + BOP_LENGTH => write!(f, "BOP_LENGTH")?, + BOP_SIZE => write!(f, "BOP_SIZE")?, + BOP_EMPTY_P => write!(f, "BOP_EMPTY_P")?, + BOP_NIL_P => write!(f, "BOP_NIL_P")?, + BOP_SUCC => write!(f, "BOP_SUCC")?, + BOP_GT => write!(f, "BOP_GT")?, + BOP_GE => write!(f, "BOP_GE")?, + BOP_NOT => write!(f, "BOP_NOT")?, + BOP_NEQ => write!(f, "BOP_NEQ")?, + BOP_MATCH => write!(f, "BOP_MATCH")?, + BOP_FREEZE => write!(f, "BOP_FREEZE")?, + BOP_UMINUS => write!(f, "BOP_UMINUS")?, + BOP_MAX => write!(f, "BOP_MAX")?, + BOP_MIN => write!(f, "BOP_MIN")?, + BOP_HASH => write!(f, "BOP_HASH")?, + BOP_CALL => write!(f, "BOP_CALL")?, + BOP_AND => write!(f, "BOP_AND")?, + BOP_OR => write!(f, "BOP_OR")?, + BOP_CMP => write!(f, "BOP_CMP")?, + BOP_DEFAULT => write!(f, "BOP_DEFAULT")?, + BOP_PACK => write!(f, "BOP_PACK")?, + BOP_INCLUDE_P => write!(f, "BOP_INCLUDE_P")?, _ => write!(f, "{bop}")?, } write!(f, ")") } - Invariant::MethodRedefined { klass, method } => { + Invariant::MethodRedefined { klass, method, cme } => { let class_name = get_class_name(klass); - write!(f, "MethodRedefined({class_name}@{:p}, {}@{:p})", + write!(f, "MethodRedefined({class_name}@{:p}, {}@{:p}, cme:{:p})", self.ptr_map.map_ptr(klass.as_ptr::<VALUE>()), method.contents_lossy(), - self.ptr_map.map_id(method.0) + self.ptr_map.map_id(method.0), + self.ptr_map.map_ptr(cme) ) } Invariant::StableConstantNames { idlist } => { @@ -199,12 +281,20 @@ impl<'a> std::fmt::Display for InvariantPrinter<'a> { } write!(f, ")") } + Invariant::NoTracePoint => write!(f, "NoTracePoint"), + Invariant::NoEPEscape(iseq) => write!(f, "NoEPEscape({})", &iseq_name(iseq)), Invariant::SingleRactorMode => write!(f, "SingleRactorMode"), + Invariant::NoSingletonClass { klass } => { + let class_name = get_class_name(klass); + write!(f, "NoSingletonClass({}@{:p})", + class_name, + self.ptr_map.map_ptr(klass.as_ptr::<VALUE>())) + } } } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Copy)] pub enum Const { Value(VALUE), CBool(bool), @@ -215,8 +305,9 @@ pub enum Const { CUInt8(u8), CUInt16(u16), CUInt32(u32), + CShape(ShapeId), CUInt64(u64), - CPtr(*mut u8), + CPtr(*const u8), CDouble(f64), } @@ -227,13 +318,67 @@ impl std::fmt::Display for Const { } impl Const { - fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> ConstPrinter<'a> { + pub fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> ConstPrinter<'a> { ConstPrinter { inner: self, ptr_map } } } +#[derive(Clone, Copy)] +pub enum RangeType { + Inclusive = 0, // include the end value + Exclusive = 1, // exclude the end value +} + +impl std::fmt::Display for RangeType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", match self { + RangeType::Inclusive => "NewRangeInclusive", + RangeType::Exclusive => "NewRangeExclusive", + }) + } +} + +impl std::fmt::Debug for RangeType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{self}") + } +} + +impl From<u32> for RangeType { + fn from(flag: u32) -> Self { + match flag { + 0 => RangeType::Inclusive, + 1 => RangeType::Exclusive, + _ => panic!("Invalid range flag: {flag}"), + } + } +} + +/// Special regex backref symbol types +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SpecialBackrefSymbol { + LastMatch, // $& + PreMatch, // $` + PostMatch, // $' + LastGroup, // $+ +} + +impl TryFrom<u8> for SpecialBackrefSymbol { + type Error = String; + + fn try_from(value: u8) -> Result<Self, Self::Error> { + match value as char { + '&' => Ok(SpecialBackrefSymbol::LastMatch), + '`' => Ok(SpecialBackrefSymbol::PreMatch), + '\'' => Ok(SpecialBackrefSymbol::PostMatch), + '+' => Ok(SpecialBackrefSymbol::LastGroup), + c => Err(format!("invalid backref symbol: '{c}'")), + } + } +} + /// Print adaptor for [`Const`]. See [`PtrPrintMap`]. -struct ConstPrinter<'a> { +pub struct ConstPrinter<'a> { inner: &'a Const, ptr_map: &'a PtrPrintMap, } @@ -242,7 +387,12 @@ impl<'a> std::fmt::Display for ConstPrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self.inner { Const::Value(val) => write!(f, "Value({})", val.print(self.ptr_map)), - Const::CPtr(val) => write!(f, "CPtr({:p})", self.ptr_map.map_ptr(val)), + // TODO: Break out CPtr as a special case. For some reason, + // when we do that now, {:p} prints a completely different + // number than {:?} does and we don't know why. + // We'll have to resolve that first. + Const::CPtr(val) => write!(f, "CPtr({:?})", self.ptr_map.map_ptr(val)), + &Const::CShape(shape_id) => write!(f, "CShape({:p})", self.ptr_map.map_shape(shape_id)), _ => write!(f, "{:?}", self.inner), } } @@ -254,7 +404,7 @@ impl<'a> std::fmt::Display for ConstPrinter<'a> { /// /// Because this is extra state external to any pointer being printed, a /// printing adapter struct that wraps the pointer along with this map is -/// required to make use of this effectly. The [`std::fmt::Display`] +/// required to make use of this effectively. The [`std::fmt::Display`] /// implementation on the adapter struct can then be reused to implement /// `Display` on the inner type with a default [`PtrPrintMap`], which /// does not perform any mapping. @@ -283,7 +433,7 @@ impl PtrPrintMap { impl PtrPrintMap { /// Map a pointer for printing - fn map_ptr<T>(&self, ptr: *const T) -> *const T { + pub fn map_ptr<T>(&self, ptr: *const T) -> *const T { // When testing, address stability is not a concern so print real address to enable code // reuse if !self.map_ptrs { @@ -311,6 +461,259 @@ impl PtrPrintMap { fn map_id(&self, id: u64) -> *const c_void { self.map_ptr(id as *const c_void) } + + /// Map an index into a Ruby object (e.g. for an ivar) for printing + fn map_index(&self, id: u64) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + fn map_offset(&self, id: i32) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + /// Map shape ID into a pointer for printing + pub fn map_shape(&self, id: ShapeId) -> *const c_void { + self.map_ptr(id.0 as *const c_void) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum SideExitReason { + UnhandledNewarraySend(vm_opt_newarray_send_type), + UnhandledDuparraySend(u64), + UnknownSpecialVariable(u64), + UnhandledHIRInsn(InsnId), + UnhandledYARVInsn(u32), + UnhandledCallType(CallType), + UnhandledBlockArg, + TooManyKeywordParameters, + FixnumAddOverflow, + FixnumSubOverflow, + FixnumMultOverflow, + FixnumLShiftOverflow, + GuardType(Type), + GuardTypeNot(Type), + GuardShape(ShapeId), + ExpandArray, + GuardNotFrozen, + GuardNotShared, + GuardLess, + GuardGreaterEq, + GuardSuperMethodEntry, + PatchPoint(Invariant), + CalleeSideExit, + ObjToStringFallback, + Interrupt, + BlockParamProxyModified, + BlockParamProxyNotIseqOrIfunc, + StackOverflow, + FixnumModByZero, + FixnumDivByZero, + BoxFixnumOverflow, +} + +#[derive(Debug, Clone, Copy)] +pub enum MethodType { + Iseq, + Cfunc, + Attrset, + Ivar, + Bmethod, + Zsuper, + Alias, + Undefined, + NotImplemented, + Optimized, + Missing, + Refined, + Null, +} + +impl From<u32> for MethodType { + fn from(value: u32) -> Self { + match value { + VM_METHOD_TYPE_ISEQ => MethodType::Iseq, + VM_METHOD_TYPE_CFUNC => MethodType::Cfunc, + VM_METHOD_TYPE_ATTRSET => MethodType::Attrset, + VM_METHOD_TYPE_IVAR => MethodType::Ivar, + VM_METHOD_TYPE_BMETHOD => MethodType::Bmethod, + VM_METHOD_TYPE_ZSUPER => MethodType::Zsuper, + VM_METHOD_TYPE_ALIAS => MethodType::Alias, + VM_METHOD_TYPE_UNDEF => MethodType::Undefined, + VM_METHOD_TYPE_NOTIMPLEMENTED => MethodType::NotImplemented, + VM_METHOD_TYPE_OPTIMIZED => MethodType::Optimized, + VM_METHOD_TYPE_MISSING => MethodType::Missing, + VM_METHOD_TYPE_REFINED => MethodType::Refined, + _ => unreachable!("unknown send_without_block def_type: {}", value), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum OptimizedMethodType { + Send, + Call, + BlockCall, + StructAref, + StructAset, +} + +impl From<u32> for OptimizedMethodType { + fn from(value: u32) -> Self { + match value { + OPTIMIZED_METHOD_TYPE_SEND => OptimizedMethodType::Send, + OPTIMIZED_METHOD_TYPE_CALL => OptimizedMethodType::Call, + OPTIMIZED_METHOD_TYPE_BLOCK_CALL => OptimizedMethodType::BlockCall, + OPTIMIZED_METHOD_TYPE_STRUCT_AREF => OptimizedMethodType::StructAref, + OPTIMIZED_METHOD_TYPE_STRUCT_ASET => OptimizedMethodType::StructAset, + _ => unreachable!("unknown send_without_block optimized method type: {}", value), + } + } +} + +impl std::fmt::Display for SideExitReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SideExitReason::UnhandledYARVInsn(opcode) => write!(f, "UnhandledYARVInsn({})", insn_name(*opcode as usize)), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MAX) => write!(f, "UnhandledNewarraySend(MAX)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MIN) => write!(f, "UnhandledNewarraySend(MIN)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_HASH) => write!(f, "UnhandledNewarraySend(HASH)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK) => write!(f, "UnhandledNewarraySend(PACK)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK_BUFFER) => write!(f, "UnhandledNewarraySend(PACK_BUFFER)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_INCLUDE_P) => write!(f, "UnhandledNewarraySend(INCLUDE_P)"), + SideExitReason::UnhandledDuparraySend(method_id) => write!(f, "UnhandledDuparraySend({method_id})"), + SideExitReason::GuardType(guard_type) => write!(f, "GuardType({guard_type})"), + SideExitReason::GuardTypeNot(guard_type) => write!(f, "GuardTypeNot({guard_type})"), + SideExitReason::GuardNotShared => write!(f, "GuardNotShared"), + SideExitReason::PatchPoint(invariant) => write!(f, "PatchPoint({invariant})"), + _ => write!(f, "{self:?}"), + } + } +} + +/// Result of resolving the receiver type for method dispatch optimization. +/// Represents whether we know the receiver's class statically at compile-time, +/// have profiled type information, or know nothing about it. +pub enum ReceiverTypeResolution { + /// No profile information available for the receiver + NoProfile, + /// The receiver has a monomorphic profile (single type observed, guard needed) + Monomorphic { profiled_type: ProfiledType }, + /// The receiver is polymorphic (multiple types, none dominant) + Polymorphic, + /// The receiver has a skewed polymorphic profile (dominant type with some other types, guard needed) + SkewedPolymorphic { profiled_type: ProfiledType }, + /// More than N types seen with no clear winner + Megamorphic, + /// Megamorphic, but with a significant skew towards one type + SkewedMegamorphic { profiled_type: ProfiledType }, + /// The receiver's class is statically known at JIT compile-time (no guard needed) + StaticallyKnown { class: VALUE }, +} + +/// Reason why a send-ish instruction cannot be optimized from a fallback instruction +#[derive(Debug, Clone, Copy)] +pub enum SendFallbackReason { + SendWithoutBlockPolymorphic, + SendWithoutBlockMegamorphic, + SendWithoutBlockNoProfiles, + SendWithoutBlockCfuncNotVariadic, + SendWithoutBlockCfuncArrayVariadic, + SendWithoutBlockNotOptimizedMethodType(MethodType), + SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType), + SendWithoutBlockNotOptimizedNeedPermission, + SendWithoutBlockBopRedefined, + SendWithoutBlockOperandsNotFixnum, + SendWithoutBlockDirectKeywordMismatch, + SendWithoutBlockDirectOptionalKeywords, + SendWithoutBlockDirectKeywordCountMismatch, + SendWithoutBlockDirectMissingKeyword, + SendPolymorphic, + SendMegamorphic, + SendNoProfiles, + SendCfuncVariadic, + SendCfuncArrayVariadic, + SendNotOptimizedMethodType(MethodType), + SendNotOptimizedNeedPermission, + CCallWithFrameTooManyArgs, + ObjToStringNotString, + TooManyArgsForLir, + /// The Proc object for a BMETHOD is not defined by an ISEQ. (See `enum rb_block_type`.) + BmethodNonIseqProc, + /// Caller supplies too few or too many arguments than what the callee's parameters expects. + ArgcParamMismatch, + /// The call has at least one feature on the caller or callee side that the optimizer does not + /// support. + ComplexArgPass, + /// Caller has keyword arguments but callee doesn't expect them; need to convert to hash. + UnexpectedKeywordArgs, + /// A singleton class has been seen for the receiver class, so we skip the optimization + /// to avoid an invalidation loop. + SingletonClassSeen, + /// The super call is passed a block that the optimizer does not support. + SuperCallWithBlock, + /// The profiled super class cannot be found. + SuperClassNotFound, + /// The `super` call uses a complex argument pattern that the optimizer does not support. + SuperComplexArgsPass, + /// The cached target of a `super` call could not be found. + SuperTargetNotFound, + /// Attempted to specialize a `super` call that doesn't have profile data. + SuperNoProfiles, + /// Cannot optimize the `super` call due to the target method. + SuperNotOptimizedMethodType(MethodType), + /// The `super` call is polymorpic. + SuperPolymorphic, + /// The `super` target call uses a complex argument pattern that the optimizer does not support. + SuperTargetComplexArgsPass, + /// Initial fallback reason for every instruction, which should be mutated to + /// a more actionable reason when an attempt to specialize the instruction fails. + Uncategorized(ruby_vminsn_type), +} + +impl Display for SendFallbackReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SendWithoutBlockPolymorphic => write!(f, "SendWithoutBlock: polymorphic call site"), + SendWithoutBlockMegamorphic => write!(f, "SendWithoutBlock: megamorphic call site"), + SendWithoutBlockNoProfiles => write!(f, "SendWithoutBlock: no profile data available"), + SendWithoutBlockCfuncNotVariadic => write!(f, "SendWithoutBlock: C function is not variadic"), + SendWithoutBlockCfuncArrayVariadic => write!(f, "SendWithoutBlock: C function expects array variadic"), + SendWithoutBlockNotOptimizedMethodType(method_type) => write!(f, "SendWithoutBlock: unsupported method type {:?}", method_type), + SendWithoutBlockNotOptimizedMethodTypeOptimized(opt_type) => write!(f, "SendWithoutBlock: unsupported optimized method type {:?}", opt_type), + SendWithoutBlockNotOptimizedNeedPermission => write!(f, "SendWithoutBlock: method private or protected and no FCALL"), + SendNotOptimizedNeedPermission => write!(f, "Send: method private or protected and no FCALL"), + SendWithoutBlockBopRedefined => write!(f, "SendWithoutBlock: basic operation was redefined"), + SendWithoutBlockOperandsNotFixnum => write!(f, "SendWithoutBlock: operands are not fixnums"), + SendWithoutBlockDirectKeywordMismatch => write!(f, "SendWithoutBlockDirect: keyword mismatch"), + SendWithoutBlockDirectOptionalKeywords => write!(f, "SendWithoutBlockDirect: optional keywords"), + SendWithoutBlockDirectKeywordCountMismatch => write!(f, "SendWithoutBlockDirect: keyword count mismatch"), + SendWithoutBlockDirectMissingKeyword => write!(f, "SendWithoutBlockDirect: missing keyword"), + SendPolymorphic => write!(f, "Send: polymorphic call site"), + SendMegamorphic => write!(f, "Send: megamorphic call site"), + SendNoProfiles => write!(f, "Send: no profile data available"), + SendCfuncVariadic => write!(f, "Send: C function is variadic"), + SendCfuncArrayVariadic => write!(f, "Send: C function expects array variadic"), + SendNotOptimizedMethodType(method_type) => write!(f, "Send: unsupported method type {:?}", method_type), + CCallWithFrameTooManyArgs => write!(f, "CCallWithFrame: too many arguments"), + ObjToStringNotString => write!(f, "ObjToString: result is not a string"), + TooManyArgsForLir => write!(f, "Too many arguments for LIR"), + BmethodNonIseqProc => write!(f, "Bmethod: Proc object is not defined by an ISEQ"), + ArgcParamMismatch => write!(f, "Argument count does not match parameter count"), + ComplexArgPass => write!(f, "Complex argument passing"), + UnexpectedKeywordArgs => write!(f, "Unexpected Keyword Args"), + SingletonClassSeen => write!(f, "Singleton class previously created for receiver class"), + SuperCallWithBlock => write!(f, "super: call made with a block"), + SuperClassNotFound => write!(f, "super: profiled class cannot be found"), + SuperComplexArgsPass => write!(f, "super: complex argument passing to `super` call"), + SuperNoProfiles => write!(f, "super: no profile data available"), + SuperNotOptimizedMethodType(method_type) => write!(f, "super: unsupported target method type {:?}", method_type), + SuperPolymorphic => write!(f, "super: polymorphic call site"), + SuperTargetNotFound => write!(f, "super: profiled target method cannot be found"), + SuperTargetComplexArgsPass => write!(f, "super: complex argument passing to `super` target call"), + Uncategorized(insn) => write!(f, "Uncategorized({})", insn_name(*insn as usize)), + } + } } /// An instruction in the SSA IR. The output of an instruction is referred to by the index of @@ -318,28 +721,133 @@ impl PtrPrintMap { /// helps with editing. #[derive(Debug, Clone)] pub enum Insn { - PutSelf, Const { val: Const }, /// SSA block parameter. Also used for function parameters in the function's entry block. - Param { idx: usize }, - - StringCopy { val: InsnId }, - StringIntern { val: InsnId }, - + Param, + + StringCopy { val: InsnId, chilled: bool, state: InsnId }, + StringIntern { val: InsnId, state: InsnId }, + StringConcat { strings: Vec<InsnId>, state: InsnId }, + /// Call rb_str_getbyte with known-Fixnum index + StringGetbyte { string: InsnId, index: InsnId }, + StringSetbyteFixnum { string: InsnId, index: InsnId, value: InsnId }, + StringAppend { recv: InsnId, other: InsnId, state: InsnId }, + StringAppendCodepoint { recv: InsnId, other: InsnId, state: InsnId }, + + /// Combine count stack values into a regexp + ToRegexp { opt: usize, values: Vec<InsnId>, state: InsnId }, + + /// Put special object (VMCORE, CBASE, etc.) based on value_type + PutSpecialObject { value_type: SpecialObjectType }, + + /// Call `to_a` on `val` if the method is defined, or make a new array `[val]` otherwise. + ToArray { val: InsnId, state: InsnId }, + /// Call `to_a` on `val` if the method is defined, or make a new array `[val]` otherwise. If we + /// called `to_a`, duplicate the returned array. + ToNewArray { val: InsnId, state: InsnId }, NewArray { elements: Vec<InsnId>, state: InsnId }, - ArraySet { array: InsnId, idx: usize, val: InsnId }, + /// NewHash contains a vec of (key, value) pairs + NewHash { elements: Vec<InsnId>, state: InsnId }, + NewRange { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, + NewRangeFixnum { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, ArrayDup { val: InsnId, state: InsnId }, + ArrayHash { elements: Vec<InsnId>, state: InsnId }, ArrayMax { elements: Vec<InsnId>, state: InsnId }, + ArrayInclude { elements: Vec<InsnId>, target: InsnId, state: InsnId }, + ArrayPackBuffer { elements: Vec<InsnId>, fmt: InsnId, buffer: InsnId, state: InsnId }, + DupArrayInclude { ary: VALUE, target: InsnId, state: InsnId }, + /// Extend `left` with the elements from `right`. `left` and `right` must both be `Array`. + ArrayExtend { left: InsnId, right: InsnId, state: InsnId }, + /// Push `val` onto `array`, where `array` is already `Array`. + ArrayPush { array: InsnId, val: InsnId, state: InsnId }, + ArrayAref { array: InsnId, index: InsnId }, + ArrayAset { array: InsnId, index: InsnId, val: InsnId }, + ArrayPop { array: InsnId, state: InsnId }, + /// Return the length of the array as a C `long` ([`types::CInt64`]) + ArrayLength { array: InsnId }, + + HashAref { hash: InsnId, key: InsnId, state: InsnId }, + HashAset { hash: InsnId, key: InsnId, val: InsnId, state: InsnId }, + HashDup { val: InsnId, state: InsnId }, + + /// Allocate an instance of the `val` object without calling `#initialize` on it. + /// This can: + /// * raise an exception if `val` is not a class + /// * run arbitrary code if `val` is a class with a custom allocator + ObjectAlloc { val: InsnId, state: InsnId }, + /// Allocate an instance of the `val` class without calling `#initialize` on it. + /// This requires that `class` has the default allocator (for example via `IsMethodCfunc`). + /// This won't raise or run arbitrary code because `class` has the default allocator. + ObjectAllocClass { class: VALUE, state: InsnId }, /// Check if the value is truthy and "return" a C boolean. In reality, we will likely fuse this /// with IfTrue/IfFalse in the backend to generate jcc. Test { val: InsnId }, - Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId }, - GetConstantPath { ic: *const iseq_inline_constant_cache }, + /// Return C `true` if `val` is `Qnil`, else `false`. + IsNil { val: InsnId }, + /// Return C `true` if `val`'s method on cd resolves to the cfunc. + IsMethodCfunc { val: InsnId, cd: *const rb_call_data, cfunc: *const u8, state: InsnId }, + /// Return C `true` if left == right + IsBitEqual { left: InsnId, right: InsnId }, + /// Return C `true` if left != right + IsBitNotEqual { left: InsnId, right: InsnId }, + /// Convert a C `bool` to a Ruby `Qtrue`/`Qfalse`. Same as `RBOOL` macro. + BoxBool { val: InsnId }, + /// Convert a C `long` to a Ruby `Fixnum`. Side exit on overflow. + BoxFixnum { val: InsnId, state: InsnId }, + UnboxFixnum { val: InsnId }, + FixnumAref { recv: InsnId, index: InsnId }, + // TODO(max): In iseq body types that are not ISEQ_TYPE_METHOD, rewrite to Constant false. + Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId, state: InsnId }, + GetConstantPath { ic: *const iseq_inline_constant_cache, state: InsnId }, + /// Kernel#block_given? but without pushing a frame. Similar to [`Insn::Defined`] with + /// `DEFINED_YIELD` + IsBlockGiven, + /// Test the bit at index of val, a Fixnum. + /// Return Qtrue if the bit is set, else Qfalse. + FixnumBitCheck { val: InsnId, index: u8 }, + /// Return Qtrue if `val` is an instance of `class`, else Qfalse. + /// Equivalent to `class_search_ancestor(CLASS_OF(val), class)`. + IsA { val: InsnId, class: InsnId }, + + /// Get a global variable named `id` + GetGlobal { id: ID, state: InsnId }, + /// Set a global variable named `id` to `val` + SetGlobal { id: ID, val: InsnId, state: InsnId }, //NewObject? - //SetIvar {}, - //GetIvar {}, + /// Get an instance variable `id` from `self_val`, using the inline cache `ic` if present + GetIvar { self_val: InsnId, id: ID, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, + /// Set `self_val`'s instance variable `id` to `val`, using the inline cache `ic` if present + SetIvar { self_val: InsnId, id: ID, val: InsnId, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, + /// Check whether an instance variable exists on `self_val` + DefinedIvar { self_val: InsnId, id: ID, pushval: VALUE, state: InsnId }, + + /// Load cfp->pc + LoadPC, + /// Load EC + LoadEC, + /// Load cfp->self + LoadSelf, + LoadField { recv: InsnId, id: ID, offset: i32, return_type: Type }, + /// Write `val` at an offset of `recv`. + /// When writing a Ruby object to a Ruby object, one must use GuardNotFrozen (or equivalent) before and WriteBarrier after. + StoreField { recv: InsnId, id: ID, offset: i32, val: InsnId }, + WriteBarrier { recv: InsnId, val: InsnId }, + + /// Get a local variable from a higher scope or the heap. + /// If `use_sp` is true, it uses the SP register to optimize the read. + /// `rest_param` is used by infer_types to infer the ArrayExact type. + GetLocal { level: u32, ep_offset: u32, use_sp: bool, rest_param: bool }, + /// Set a local variable in a higher scope or the heap + SetLocal { level: u32, ep_offset: u32, val: InsnId }, + GetSpecialSymbol { symbol_type: SpecialBackrefSymbol, state: InsnId }, + GetSpecialNumber { nth: u64, state: InsnId }, + + /// Get a class variable `id` + GetClassVar { id: ID, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + /// Set a class variable `id` to `val` + SetClassVar { id: ID, val: InsnId, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, /// Own a FrameState so that instructions can look up their dominating FrameState when /// generating deopt side-exits and frame reconstruction metadata. Does not directly generate @@ -353,20 +861,113 @@ pub enum Insn { IfTrue { val: InsnId, target: BranchEdge }, IfFalse { val: InsnId, target: BranchEdge }, - /// Call a C function + /// Call a C function without pushing a frame /// `name` is for printing purposes only - CCall { cfun: *const u8, args: Vec<InsnId>, name: ID, return_type: Type, elidable: bool }, + CCall { cfunc: *const u8, recv: InsnId, args: Vec<InsnId>, name: ID, return_type: Type, elidable: bool }, + + /// Call a C function that pushes a frame + CCallWithFrame { + cd: *const rb_call_data, // cd for falling back to SendWithoutBlock + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + blockiseq: Option<IseqPtr>, + }, - /// Send without block with dynamic dispatch + /// Call a variadic C function with signature: func(int argc, VALUE *argv, VALUE recv) + /// This handles frame setup, argv creation, and frame teardown all in one + CCallVariadic { + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + blockiseq: Option<IseqPtr>, + }, + + /// Un-optimized fallback implementation (dynamic dispatch) for send-ish instructions /// Ignoring keyword arguments etc for now - SendWithoutBlock { self_val: InsnId, call_info: CallInfo, cd: *const rb_call_data, args: Vec<InsnId>, state: InsnId }, - Send { self_val: InsnId, call_info: CallInfo, cd: *const rb_call_data, blockiseq: IseqPtr, args: Vec<InsnId>, state: InsnId }, - SendWithoutBlockDirect { self_val: InsnId, call_info: CallInfo, cd: *const rb_call_data, iseq: IseqPtr, args: Vec<InsnId>, state: InsnId }, + SendWithoutBlock { + recv: InsnId, + cd: *const rb_call_data, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + Send { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + SendForward { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeSuper { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeBlock { + cd: *const rb_call_data, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + /// Call Proc#call optimized method type. + InvokeProc { + recv: InsnId, + args: Vec<InsnId>, + state: InsnId, + kw_splat: bool, + }, + + /// Optimized ISEQ call + SendWithoutBlockDirect { + recv: InsnId, + cd: *const rb_call_data, + cme: *const rb_callable_method_entry_t, + iseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + }, + + // Invoke a builtin function + InvokeBuiltin { + bf: rb_builtin_function, + recv: InsnId, + args: Vec<InsnId>, + state: InsnId, + leaf: bool, + return_type: Option<Type>, // None for unannotated builtins + }, + /// Set up frame. Remember the address as the JIT entry for the insn_idx in `jit_entry_insns()[jit_entry_idx]`. + EntryPoint { jit_entry_idx: Option<usize> }, /// Control flow instructions Return { val: InsnId }, + /// Non-local control flow. See the throw YARV instruction + Throw { throw_state: u32, val: InsnId, state: InsnId }, - /// Fixnum +, -, *, /, %, ==, !=, <, <=, >, >= + /// Fixnum +, -, *, /, %, ==, !=, <, <=, >, >=, &, |, ^, << FixnumAdd { left: InsnId, right: InsnId, state: InsnId }, FixnumSub { left: InsnId, right: InsnId, state: InsnId }, FixnumMult { left: InsnId, right: InsnId, state: InsnId }, @@ -378,24 +979,72 @@ pub enum Insn { FixnumLe { left: InsnId, right: InsnId }, FixnumGt { left: InsnId, right: InsnId }, FixnumGe { left: InsnId, right: InsnId }, + FixnumAnd { left: InsnId, right: InsnId }, + FixnumOr { left: InsnId, right: InsnId }, + FixnumXor { left: InsnId, right: InsnId }, + FixnumLShift { left: InsnId, right: InsnId, state: InsnId }, + FixnumRShift { left: InsnId, right: InsnId }, + + // Distinct from `SendWithoutBlock` with `mid:to_s` because does not have a patch point for String to_s being redefined + ObjToString { val: InsnId, cd: *const rb_call_data, state: InsnId }, + AnyToString { val: InsnId, str: InsnId, state: InsnId }, /// Side-exit if val doesn't have the expected type. GuardType { val: InsnId, guard_type: Type, state: InsnId }, - /// Side-exit if val is not the expected VALUE. - GuardBitEquals { val: InsnId, expected: VALUE, state: InsnId }, + GuardTypeNot { val: InsnId, guard_type: Type, state: InsnId }, + /// Side-exit if val is not the expected Const. + GuardBitEquals { val: InsnId, expected: Const, reason: SideExitReason, state: InsnId }, + /// Side-exit if val doesn't have the expected shape. + GuardShape { val: InsnId, shape: ShapeId, state: InsnId }, + /// Side-exit if the block param has been modified or the block handler for the frame + /// is neither ISEQ nor ifunc, which makes it incompatible with rb_block_param_proxy. + GuardBlockParamProxy { level: u32, state: InsnId }, + /// Side-exit if val is frozen. Does *not* check if the val is an immediate; assumes that it is + /// a heap object. + GuardNotFrozen { recv: InsnId, state: InsnId }, + /// Side-exit if val is shared. Does *not* check if the val is an immediate; assumes + /// that it is a heap object. + GuardNotShared { recv: InsnId, state: InsnId }, + /// Side-exit if left is not greater than or equal to right (both operands are C long). + GuardGreaterEq { left: InsnId, right: InsnId, state: InsnId }, + /// Side-exit if left is not less than right (both operands are C long). + GuardLess { left: InsnId, right: InsnId, state: InsnId }, + /// Side-exit if the method entry at ep[VM_ENV_DATA_INDEX_ME_CREF] doesn't match the expected CME. + /// Used to ensure super calls are made from the expected method context. + GuardSuperMethodEntry { cme: *const rb_callable_method_entry_t, state: InsnId }, + /// Get the block handler from ep[VM_ENV_DATA_INDEX_SPECVAL] at the local EP (LEP). + GetBlockHandler, /// Generate no code (or padding if necessary) and insert a patch point /// that can be rewritten to a side exit when the Invariant is broken. - PatchPoint(Invariant), + PatchPoint { invariant: Invariant, state: InsnId }, + + /// Side-exit into the interpreter. + SideExit { state: InsnId, reason: SideExitReason }, + + /// Increment a counter in ZJIT stats + IncrCounter(Counter), + + /// Increment a counter in ZJIT stats for the given counter pointer + IncrCounterPtr { counter_ptr: *mut u64 }, + + /// Equivalent of RUBY_VM_CHECK_INTS. Automatically inserted by the compiler before jumps and + /// return instructions. + CheckInterrupts { state: InsnId }, } impl Insn { /// Not every instruction returns a value. Return true if the instruction does and false otherwise. pub fn has_output(&self) -> bool { match self { - Insn::ArraySet { .. } | Insn::Snapshot { .. } | Insn::Jump(_) - | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::Return { .. } - | Insn::PatchPoint { .. } => false, + Insn::Jump(_) + | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::EntryPoint { .. } | Insn::Return { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } + | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetGlobal { .. } + | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } + | Insn::ArrayAset { .. } => false, _ => true, } } @@ -403,42 +1052,177 @@ impl Insn { /// Return true if the instruction ends a basic block and false otherwise. pub fn is_terminator(&self) -> bool { match self { - Insn::Jump(_) | Insn::Return { .. } => true, + Insn::Jump(_) | Insn::Return { .. } | Insn::SideExit { .. } | Insn::Throw { .. } => true, _ => false, } } - pub fn print<'a>(&self, ptr_map: &'a PtrPrintMap) -> InsnPrinter<'a> { - InsnPrinter { inner: self.clone(), ptr_map } + pub fn print<'a>(&self, ptr_map: &'a PtrPrintMap, iseq: Option<IseqPtr>) -> InsnPrinter<'a> { + InsnPrinter { inner: self.clone(), ptr_map, iseq } + } + + // TODO(Jacob): Model SP. ie, all allocations modify stack size but using the effect for stack modification feels excessive + // TODO(Jacob): Add sideeffect failure bit + fn effects_of(&self) -> Effect { + const allocates: Effect = Effect::read_write(abstract_heaps::PC.union(abstract_heaps::Allocator), abstract_heaps::Allocator); + match &self { + Insn::Const { .. } => effects::Empty, + Insn::Param { .. } => effects::Empty, + Insn::StringCopy { .. } => allocates, + Insn::StringIntern { .. } => effects::Any, + Insn::StringConcat { .. } => effects::Any, + Insn::StringGetbyte { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::StringSetbyteFixnum { .. } => effects::Any, + Insn::StringAppend { .. } => effects::Any, + Insn::StringAppendCodepoint { .. } => effects::Any, + Insn::ToRegexp { .. } => effects::Any, + Insn::PutSpecialObject { .. } => effects::Any, + Insn::ToArray { .. } => effects::Any, + Insn::ToNewArray { .. } => effects::Any, + Insn::NewArray { .. } => allocates, + Insn::NewHash { elements, .. } => { + // NewHash's operands may be hashed and compared for equality, which could have + // side-effects. Empty hashes are definitely elidable. + if elements.is_empty() { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::NewRange { .. } => effects::Any, + Insn::NewRangeFixnum { .. } => allocates, + Insn::ArrayDup { .. } => allocates, + Insn::ArrayHash { .. } => effects::Any, + Insn::ArrayMax { .. } => effects::Any, + Insn::ArrayInclude { .. } => effects::Any, + Insn::ArrayPackBuffer { .. } => effects::Any, + Insn::DupArrayInclude { .. } => effects::Any, + Insn::ArrayExtend { .. } => effects::Any, + Insn::ArrayPush { .. } => effects::Any, + Insn::ArrayAref { .. } => effects::Any, + Insn::ArrayAset { .. } => effects::Any, + Insn::ArrayPop { .. } => effects::Any, + Insn::ArrayLength { .. } => Effect::write(abstract_heaps::Empty), + Insn::HashAref { .. } => effects::Any, + Insn::HashAset { .. } => effects::Any, + Insn::HashDup { .. } => allocates, + Insn::ObjectAlloc { .. } => effects::Any, + Insn::ObjectAllocClass { .. } => allocates, + Insn::Test { .. } => effects::Empty, + Insn::IsNil { .. } => effects::Empty, + Insn::IsMethodCfunc { .. } => effects::Any, + Insn::IsBitEqual { .. } => effects::Empty, + Insn::IsBitNotEqual { .. } => effects::Any, + Insn::BoxBool { .. } => effects::Empty, + Insn::BoxFixnum { .. } => effects::Empty, + Insn::UnboxFixnum { .. } => effects::Any, + Insn::FixnumAref { .. } => effects::Empty, + Insn::Defined { .. } => effects::Any, + Insn::GetConstantPath { .. } => effects::Any, + Insn::IsBlockGiven { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::FixnumBitCheck { .. } => effects::Any, + Insn::IsA { .. } => effects::Empty, + Insn::GetGlobal { .. } => effects::Any, + Insn::SetGlobal { .. } => effects::Any, + Insn::GetIvar { .. } => effects::Any, + Insn::SetIvar { .. } => effects::Any, + Insn::DefinedIvar { .. } => effects::Any, + Insn::LoadPC { .. } => Effect::read_write(abstract_heaps::PC, abstract_heaps::Empty), + Insn::LoadEC { .. } => effects::Empty, + Insn::LoadSelf { .. } => Effect::read_write(abstract_heaps::Frame, abstract_heaps::Empty), + Insn::LoadField { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::StoreField { .. } => effects::Any, + Insn::WriteBarrier { .. } => effects::Any, + Insn::GetLocal { .. } => Effect::read_write(abstract_heaps::Locals, abstract_heaps::Empty), + Insn::SetLocal { .. } => effects::Any, + Insn::GetSpecialSymbol { .. } => effects::Any, + Insn::GetSpecialNumber { .. } => effects::Any, + Insn::GetClassVar { .. } => effects::Any, + Insn::SetClassVar { .. } => effects::Any, + Insn::Snapshot { .. } => effects::Empty, + Insn::Jump(_) => effects::Any, + Insn::IfTrue { .. } => effects::Any, + Insn::IfFalse { .. } => effects::Any, + Insn::CCall { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallWithFrame { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallVariadic { .. } => effects::Any, + Insn::SendWithoutBlock { .. } => effects::Any, + Insn::Send { .. } => effects::Any, + Insn::SendForward { .. } => effects::Any, + Insn::InvokeSuper { .. } => effects::Any, + Insn::InvokeBlock { .. } => effects::Any, + Insn::SendWithoutBlockDirect { .. } => effects::Any, + Insn::InvokeBuiltin { .. } => effects::Any, + Insn::EntryPoint { .. } => effects::Any, + Insn::Return { .. } => effects::Any, + Insn::Throw { .. } => effects::Any, + Insn::FixnumAdd { .. } => effects::Empty, + Insn::FixnumSub { .. } => effects::Empty, + Insn::FixnumMult { .. } => effects::Empty, + Insn::FixnumDiv { .. } => effects::Any, + Insn::FixnumMod { .. } => effects::Any, + Insn::FixnumEq { .. } => effects::Empty, + Insn::FixnumNeq { .. } => effects::Empty, + Insn::FixnumLt { .. } => effects::Empty, + Insn::FixnumLe { .. } => effects::Empty, + Insn::FixnumGt { .. } => effects::Empty, + Insn::FixnumGe { .. } => effects::Empty, + Insn::FixnumAnd { .. } => effects::Empty, + Insn::FixnumOr { .. } => effects::Empty, + Insn::FixnumXor { .. } => effects::Empty, + Insn::FixnumLShift { .. } => effects::Empty, + Insn::FixnumRShift { .. } => effects::Empty, + Insn::ObjToString { .. } => effects::Any, + Insn::AnyToString { .. } => effects::Any, + Insn::GuardType { .. } => effects::Any, + Insn::GuardTypeNot { .. } => effects::Any, + Insn::GuardBitEquals { .. } => effects::Any, + Insn::GuardShape { .. } => effects::Any, + Insn::GuardBlockParamProxy { .. } => effects::Any, + Insn::GuardNotFrozen { .. } => effects::Any, + Insn::GuardNotShared { .. } => effects::Any, + Insn::GuardGreaterEq { .. } => effects::Any, + Insn::GuardSuperMethodEntry { .. } => effects::Any, + Insn::GetBlockHandler { .. } => effects::Any, + Insn::GuardLess { .. } => effects::Any, + Insn::PatchPoint { .. } => effects::Any, + Insn::SideExit { .. } => effects::Any, + Insn::IncrCounter(_) => effects::Any, + Insn::IncrCounterPtr { .. } => effects::Any, + Insn::CheckInterrupts { .. } => effects::Any, + Insn::InvokeProc { .. } => effects::Any, + } } - /// Return true if the instruction needs to be kept around. For example, if the instruction - /// might have a side effect, or if the instruction may raise an exception. - fn has_effects(&self) -> bool { - match self { - Insn::PutSelf => false, - Insn::Const { .. } => false, - Insn::Param { .. } => false, - Insn::StringCopy { .. } => false, - Insn::NewArray { .. } => false, - Insn::ArrayDup { .. } => false, - Insn::Test { .. } => false, - Insn::Snapshot { .. } => false, - Insn::FixnumAdd { .. } => false, - Insn::FixnumSub { .. } => false, - Insn::FixnumMult { .. } => false, - // TODO(max): Consider adding a Guard that the rhs is non-zero before Div and Mod - // Div *is* critical unless we can prove the right hand side != 0 - // Mod *is* critical unless we can prove the right hand side != 0 - Insn::FixnumEq { .. } => false, - Insn::FixnumNeq { .. } => false, - Insn::FixnumLt { .. } => false, - Insn::FixnumLe { .. } => false, - Insn::FixnumGt { .. } => false, - Insn::FixnumGe { .. } => false, - Insn::CCall { elidable, .. } => !elidable, - _ => true, - } + /// Return true if we can safely omit the instruction. This occurs when one of the following + /// conditions are met. + /// 1. The instruction does not write anything. + /// 2. The instruction only allocates and writes nothing else. + /// Calling the effects of our instruction `insn_effects`, we need: + /// `effects::Empty` to include `insn_effects.write` or `effects::Allocator` to include + /// `insn_effects.write`. + /// We can simplify this to `effects::Empty.union(effects::Allocator).includes(insn_effects.write)`. + /// But the union of `Allocator` and `Empty` is simply `Allocator`, so our entire function + /// collapses to `effects::Allocator.includes(insn_effects.write)`. + /// Note: These are restrictions on the `write` `EffectSet` only. Even instructions with + /// `read: effects::Any` could potentially be omitted. + fn is_elidable(&self) -> bool { + abstract_heaps::Allocator.includes(self.effects_of().write_bits()) } } @@ -446,13 +1230,45 @@ impl Insn { pub struct InsnPrinter<'a> { inner: Insn, ptr_map: &'a PtrPrintMap, + iseq: Option<IseqPtr>, +} + +/// Get the name of a local variable given iseq, level, and ep_offset. +/// Returns +/// - `":name"` if iseq is available and name is a real identifier, +/// - `"<empty>"` for anonymous locals. +/// - `None` if iseq is not available. +/// (When `Insn` is printed in a panic/debug message the `Display::fmt` method is called, which can't access an iseq.) +/// +/// This mimics local_var_name() from iseq.c. +fn get_local_var_name_for_printer(iseq: Option<IseqPtr>, level: u32, ep_offset: u32) -> Option<String> { + let mut current_iseq = iseq?; + for _ in 0..level { + current_iseq = unsafe { rb_get_iseq_body_parent_iseq(current_iseq) }; + } + let local_idx = ep_offset_to_local_idx(current_iseq, ep_offset); + let id: ID = unsafe { rb_zjit_local_id(current_iseq, local_idx.try_into().unwrap()) }; + + if id.0 == 0 || unsafe { rb_id2str(id) } == Qfalse { + return Some(String::from("<empty>")); + } + + Some(format!(":{}", id.contents_lossy())) } +static REGEXP_FLAGS: &[(u32, &str)] = &[ + (ONIG_OPTION_MULTILINE, "MULTILINE"), + (ONIG_OPTION_IGNORECASE, "IGNORECASE"), + (ONIG_OPTION_EXTEND, "EXTENDED"), + (ARG_ENCODING_FIXED, "FIXEDENCODING"), + (ARG_ENCODING_NONE, "NOENCODING"), +]; + impl<'a> std::fmt::Display for InsnPrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match &self.inner { Insn::Const { val } => { write!(f, "Const {}", val.print(self.ptr_map)) } - Insn::Param { idx } => { write!(f, "Param {idx}") } + Insn::Param => { write!(f, "Param") } Insn::NewArray { elements, .. } => { write!(f, "NewArray")?; let mut prefix = " "; @@ -462,6 +1278,35 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { } Ok(()) } + Insn::ArrayAref { array, index, .. } => { + write!(f, "ArrayAref {array}, {index}") + } + Insn::ArrayAset { array, index, val, ..} => { + write!(f, "ArrayAset {array}, {index}, {val}") + } + Insn::ArrayPop { array, .. } => { + write!(f, "ArrayPop {array}") + } + Insn::ArrayLength { array } => { + write!(f, "ArrayLength {array}") + } + Insn::NewHash { elements, .. } => { + write!(f, "NewHash")?; + let mut prefix = " "; + for chunk in elements.chunks(2) { + if let [key, value] = chunk { + write!(f, "{prefix}{key}: {value}")?; + prefix = ", "; + } + } + Ok(()) + } + Insn::NewRange { low, high, flag, .. } => { + write!(f, "NewRange {low} {flag} {high}") + } + Insn::NewRangeFixnum { low, high, flag, .. } => { + write!(f, "NewRangeFixnum {low} {flag} {high}") + } Insn::ArrayMax { elements, .. } => { write!(f, "ArrayMax")?; let mut prefix = " "; @@ -471,37 +1316,173 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { } Ok(()) } - Insn::ArraySet { array, idx, val } => { write!(f, "ArraySet {array}, {idx}, {val}") } + Insn::ArrayHash { elements, .. } => { + write!(f, "ArrayHash")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayInclude { elements, target, .. } => { + write!(f, "ArrayInclude")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + write!(f, " | {target}") + } + Insn::ArrayPackBuffer { elements, fmt, buffer, .. } => { + write!(f, "ArrayPackBuffer ")?; + for element in elements { + write!(f, "{element}, ")?; + } + write!(f, "fmt: {fmt}, buf: {buffer}") + } + Insn::DupArrayInclude { ary, target, .. } => { + write!(f, "DupArrayInclude {} | {}", ary.print(self.ptr_map), target) + } Insn::ArrayDup { val, .. } => { write!(f, "ArrayDup {val}") } - Insn::StringCopy { val } => { write!(f, "StringCopy {val}") } + Insn::HashDup { val, .. } => { write!(f, "HashDup {val}") } + Insn::HashAref { hash, key, .. } => { write!(f, "HashAref {hash}, {key}")} + Insn::HashAset { hash, key, val, .. } => { write!(f, "HashAset {hash}, {key}, {val}")} + Insn::ObjectAlloc { val, .. } => { write!(f, "ObjectAlloc {val}") } + &Insn::ObjectAllocClass { class, .. } => { + let class_name = get_class_name(class); + write!(f, "ObjectAllocClass {class_name}:{}", class.print(self.ptr_map)) + } + Insn::StringCopy { val, .. } => { write!(f, "StringCopy {val}") } + Insn::StringConcat { strings, .. } => { + write!(f, "StringConcat")?; + let mut prefix = " "; + for string in strings { + write!(f, "{prefix}{string}")?; + prefix = ", "; + } + + Ok(()) + } + Insn::StringGetbyte { string, index, .. } => { + write!(f, "StringGetbyte {string}, {index}") + } + Insn::StringSetbyteFixnum { string, index, value, .. } => { + write!(f, "StringSetbyteFixnum {string}, {index}, {value}") + } + Insn::StringAppend { recv, other, .. } => { + write!(f, "StringAppend {recv}, {other}") + } + Insn::StringAppendCodepoint { recv, other, .. } => { + write!(f, "StringAppendCodepoint {recv}, {other}") + } + Insn::ToRegexp { values, opt, .. } => { + write!(f, "ToRegexp")?; + let mut prefix = " "; + for value in values { + write!(f, "{prefix}{value}")?; + prefix = ", "; + } + + let opt = *opt as u32; + if opt != 0 { + write!(f, ", ")?; + let mut sep = ""; + for (flag, name) in REGEXP_FLAGS { + if opt & flag != 0 { + write!(f, "{sep}{name}")?; + sep = "|"; + } + } + } + + Ok(()) + } Insn::Test { val } => { write!(f, "Test {val}") } + Insn::IsNil { val } => { write!(f, "IsNil {val}") } + Insn::IsMethodCfunc { val, cd, .. } => { write!(f, "IsMethodCFunc {val}, :{}", ruby_call_method_name(*cd)) } + Insn::IsBitEqual { left, right } => write!(f, "IsBitEqual {left}, {right}"), + Insn::IsBitNotEqual { left, right } => write!(f, "IsBitNotEqual {left}, {right}"), + Insn::BoxBool { val } => write!(f, "BoxBool {val}"), + Insn::BoxFixnum { val, .. } => write!(f, "BoxFixnum {val}"), + Insn::UnboxFixnum { val } => write!(f, "UnboxFixnum {val}"), + Insn::FixnumAref { recv, index } => write!(f, "FixnumAref {recv}, {index}"), Insn::Jump(target) => { write!(f, "Jump {target}") } Insn::IfTrue { val, target } => { write!(f, "IfTrue {val}, {target}") } Insn::IfFalse { val, target } => { write!(f, "IfFalse {val}, {target}") } - Insn::SendWithoutBlock { self_val, call_info, args, .. } => { - write!(f, "SendWithoutBlock {self_val}, :{}", call_info.method_name)?; + Insn::SendWithoutBlock { recv, cd, args, reason, .. } => { + write!(f, "SendWithoutBlock {recv}, :{}", ruby_call_method_name(*cd))?; for arg in args { write!(f, ", {arg}")?; } + write!(f, " # SendFallbackReason: {reason}")?; Ok(()) } - Insn::SendWithoutBlockDirect { self_val, call_info, iseq, args, .. } => { - write!(f, "SendWithoutBlockDirect {self_val}, :{} ({:?})", call_info.method_name, self.ptr_map.map_ptr(iseq))?; + Insn::SendWithoutBlockDirect { recv, cd, iseq, args, .. } => { + write!(f, "SendWithoutBlockDirect {recv}, :{} ({:?})", ruby_call_method_name(*cd), self.ptr_map.map_ptr(iseq))?; for arg in args { write!(f, ", {arg}")?; } Ok(()) } - Insn::Send { self_val, call_info, args, blockiseq, .. } => { + Insn::Send { recv, cd, args, blockiseq, reason, .. } => { // For tests, we want to check HIR snippets textually. Addresses change // between runs, making tests fail. Instead, pick an arbitrary hex value to // use as a "pointer" so we can check the rest of the HIR. - write!(f, "Send {self_val}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), call_info.method_name)?; + write!(f, "Send {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::SendForward { recv, cd, args, blockiseq, reason, .. } => { + write!(f, "SendForward {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeSuper { recv, blockiseq, args, reason, .. } => { + write!(f, "InvokeSuper {recv}, {:p}", self.ptr_map.map_ptr(blockiseq))?; for arg in args { write!(f, ", {arg}")?; } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeBlock { args, reason, .. } => { + write!(f, "InvokeBlock")?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeProc { recv, args, kw_splat, .. } => { + write!(f, "InvokeProc {recv}")?; + for arg in args { + write!(f, ", {arg}")?; + } + if *kw_splat { + write!(f, ", kw_splat")?; + } Ok(()) } + Insn::InvokeBuiltin { bf, args, leaf, .. } => { + let bf_name = unsafe { CStr::from_ptr(bf.name) }.to_str().unwrap(); + write!(f, "InvokeBuiltin{} {}", + if *leaf { " leaf" } else { "" }, + // e.g. Code that use `Primitive.cexpr!`. From BUILTIN_INLINE_PREFIX. + if bf_name.starts_with("_bi") { "<inline_expr>" } else { bf_name })?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } + &Insn::EntryPoint { jit_entry_idx: Some(idx) } => write!(f, "EntryPoint JIT({idx})"), + &Insn::EntryPoint { jit_entry_idx: None } => write!(f, "EntryPoint interpreter"), Insn::Return { val } => { write!(f, "Return {val}") } Insn::FixnumAdd { left, right, .. } => { write!(f, "FixnumAdd {left}, {right}") }, Insn::FixnumSub { left, right, .. } => { write!(f, "FixnumSub {left}, {right}") }, @@ -514,44 +1495,153 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::FixnumLe { left, right, .. } => { write!(f, "FixnumLe {left}, {right}") }, Insn::FixnumGt { left, right, .. } => { write!(f, "FixnumGt {left}, {right}") }, Insn::FixnumGe { left, right, .. } => { write!(f, "FixnumGe {left}, {right}") }, + Insn::FixnumAnd { left, right, .. } => { write!(f, "FixnumAnd {left}, {right}") }, + Insn::FixnumOr { left, right, .. } => { write!(f, "FixnumOr {left}, {right}") }, + Insn::FixnumXor { left, right, .. } => { write!(f, "FixnumXor {left}, {right}") }, + Insn::FixnumLShift { left, right, .. } => { write!(f, "FixnumLShift {left}, {right}") }, + Insn::FixnumRShift { left, right, .. } => { write!(f, "FixnumRShift {left}, {right}") }, Insn::GuardType { val, guard_type, .. } => { write!(f, "GuardType {val}, {}", guard_type.print(self.ptr_map)) }, + Insn::GuardTypeNot { val, guard_type, .. } => { write!(f, "GuardTypeNot {val}, {}", guard_type.print(self.ptr_map)) }, Insn::GuardBitEquals { val, expected, .. } => { write!(f, "GuardBitEquals {val}, {}", expected.print(self.ptr_map)) }, - Insn::PatchPoint(invariant) => { write!(f, "PatchPoint {}", invariant.print(self.ptr_map)) }, - Insn::GetConstantPath { ic } => { write!(f, "GetConstantPath {:p}", self.ptr_map.map_ptr(ic)) }, - Insn::CCall { cfun, args, name, return_type: _, elidable: _ } => { - write!(f, "CCall {}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfun))?; + &Insn::GuardShape { val, shape, .. } => { write!(f, "GuardShape {val}, {:p}", self.ptr_map.map_shape(shape)) }, + Insn::GuardBlockParamProxy { level, .. } => write!(f, "GuardBlockParamProxy l{level}"), + Insn::GuardNotFrozen { recv, .. } => write!(f, "GuardNotFrozen {recv}"), + Insn::GuardNotShared { recv, .. } => write!(f, "GuardNotShared {recv}"), + Insn::GuardLess { left, right, .. } => write!(f, "GuardLess {left}, {right}"), + Insn::GuardGreaterEq { left, right, .. } => write!(f, "GuardGreaterEq {left}, {right}"), + Insn::GuardSuperMethodEntry { cme, .. } => write!(f, "GuardSuperMethodEntry {:p}", self.ptr_map.map_ptr(cme)), + Insn::GetBlockHandler => write!(f, "GetBlockHandler"), + Insn::PatchPoint { invariant, .. } => { write!(f, "PatchPoint {}", invariant.print(self.ptr_map)) }, + Insn::GetConstantPath { ic, .. } => { write!(f, "GetConstantPath {:p}", self.ptr_map.map_ptr(ic)) }, + Insn::IsBlockGiven => { write!(f, "IsBlockGiven") }, + Insn::FixnumBitCheck {val, index} => { write!(f, "FixnumBitCheck {val}, {index}") }, + Insn::CCall { cfunc, recv, args, name, return_type: _, elidable: _ } => { + write!(f, "CCall {recv}, :{}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfunc))?; for arg in args { write!(f, ", {arg}")?; } Ok(()) }, - Insn::Snapshot { state } => write!(f, "Snapshot {}", state), - insn => { write!(f, "{insn:?}") } + Insn::CCallWithFrame { cfunc, recv, args, name, blockiseq, .. } => { + write!(f, "CCallWithFrame {recv}, :{}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + if let Some(blockiseq) = blockiseq { + write!(f, ", block={:p}", self.ptr_map.map_ptr(blockiseq))?; + } + Ok(()) + }, + Insn::CCallVariadic { cfunc, recv, args, name, .. } => { + write!(f, "CCallVariadic {recv}, :{}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + }, + Insn::IncrCounterPtr { .. } => write!(f, "IncrCounterPtr"), + Insn::Snapshot { state } => write!(f, "Snapshot {}", state.print(self.ptr_map)), + Insn::Defined { op_type, v, .. } => { + // op_type (enum defined_type) printing logic from iseq.c. + // Not sure why rb_iseq_defined_string() isn't exhaustive. + write!(f, "Defined ")?; + let op_type = *op_type as u32; + if op_type == DEFINED_FUNC { + write!(f, "func")?; + } else if op_type == DEFINED_REF { + write!(f, "ref")?; + } else if op_type == DEFINED_CONST_FROM { + write!(f, "constant-from")?; + } else { + write!(f, "{}", String::from_utf8_lossy(unsafe { rb_iseq_defined_string(op_type).as_rstring_byte_slice().unwrap() }))?; + }; + write!(f, ", {v}") + } + Insn::DefinedIvar { self_val, id, .. } => write!(f, "DefinedIvar {self_val}, :{}", id.contents_lossy()), + Insn::GetIvar { self_val, id, .. } => write!(f, "GetIvar {self_val}, :{}", id.contents_lossy()), + Insn::LoadPC => write!(f, "LoadPC"), + Insn::LoadEC => write!(f, "LoadEC"), + Insn::LoadSelf => write!(f, "LoadSelf"), + &Insn::LoadField { recv, id, offset, return_type: _ } => write!(f, "LoadField {recv}, :{}@{:p}", id.contents_lossy(), self.ptr_map.map_offset(offset)), + &Insn::StoreField { recv, id, offset, val } => write!(f, "StoreField {recv}, :{}@{:p}, {val}", id.contents_lossy(), self.ptr_map.map_offset(offset)), + &Insn::WriteBarrier { recv, val } => write!(f, "WriteBarrier {recv}, {val}"), + Insn::SetIvar { self_val, id, val, .. } => write!(f, "SetIvar {self_val}, :{}, {val}", id.contents_lossy()), + Insn::GetGlobal { id, .. } => write!(f, "GetGlobal :{}", id.contents_lossy()), + Insn::SetGlobal { id, val, .. } => write!(f, "SetGlobal :{}, {val}", id.contents_lossy()), + &Insn::GetLocal { level, ep_offset, use_sp: true, rest_param } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "GetLocal {name}l{level}, SP@{}{}", ep_offset + 1, if rest_param { ", *" } else { "" }) + }, + &Insn::GetLocal { level, ep_offset, use_sp: false, rest_param } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "GetLocal {name}l{level}, EP@{ep_offset}{}", if rest_param { ", *" } else { "" }) + }, + &Insn::SetLocal { val, level, ep_offset } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "SetLocal {name}l{level}, EP@{ep_offset}, {val}") + }, + Insn::GetSpecialSymbol { symbol_type, .. } => write!(f, "GetSpecialSymbol {symbol_type:?}"), + Insn::GetSpecialNumber { nth, .. } => write!(f, "GetSpecialNumber {nth}"), + Insn::GetClassVar { id, .. } => write!(f, "GetClassVar :{}", id.contents_lossy()), + Insn::SetClassVar { id, val, .. } => write!(f, "SetClassVar :{}, {val}", id.contents_lossy()), + Insn::ToArray { val, .. } => write!(f, "ToArray {val}"), + Insn::ToNewArray { val, .. } => write!(f, "ToNewArray {val}"), + Insn::ArrayExtend { left, right, .. } => write!(f, "ArrayExtend {left}, {right}"), + Insn::ArrayPush { array, val, .. } => write!(f, "ArrayPush {array}, {val}"), + Insn::ObjToString { val, .. } => { write!(f, "ObjToString {val}") }, + Insn::StringIntern { val, .. } => { write!(f, "StringIntern {val}") }, + Insn::AnyToString { val, str, .. } => { write!(f, "AnyToString {val}, str: {str}") }, + Insn::SideExit { reason, .. } => write!(f, "SideExit {reason}"), + Insn::PutSpecialObject { value_type } => write!(f, "PutSpecialObject {value_type}"), + Insn::Throw { throw_state, val, .. } => { + write!(f, "Throw ")?; + match throw_state & VM_THROW_STATE_MASK { + RUBY_TAG_NONE => write!(f, "TAG_NONE"), + RUBY_TAG_RETURN => write!(f, "TAG_RETURN"), + RUBY_TAG_BREAK => write!(f, "TAG_BREAK"), + RUBY_TAG_NEXT => write!(f, "TAG_NEXT"), + RUBY_TAG_RETRY => write!(f, "TAG_RETRY"), + RUBY_TAG_REDO => write!(f, "TAG_REDO"), + RUBY_TAG_RAISE => write!(f, "TAG_RAISE"), + RUBY_TAG_THROW => write!(f, "TAG_THROW"), + RUBY_TAG_FATAL => write!(f, "TAG_FATAL"), + tag => write!(f, "{tag}") + }?; + if throw_state & VM_THROW_NO_ESCAPE_FLAG != 0 { + write!(f, "|NO_ESCAPE")?; + } + write!(f, ", {val}") + } + Insn::IncrCounter(counter) => write!(f, "IncrCounter {counter:?}"), + Insn::CheckInterrupts { .. } => write!(f, "CheckInterrupts"), + Insn::IsA { val, class } => write!(f, "IsA {val}, {class}"), } } } impl std::fmt::Display for Insn { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - self.print(&PtrPrintMap::identity()).fmt(f) + self.print(&PtrPrintMap::identity(), None).fmt(f) } } /// An extended basic block in a [`Function`]. #[derive(Default, Debug)] pub struct Block { + /// The index of the first YARV instruction for the Block in the ISEQ + pub insn_idx: u32, params: Vec<InsnId>, insns: Vec<InsnId>, } impl Block { /// Return an iterator over params - pub fn params(&self) -> Iter<InsnId> { + pub fn params(&self) -> Iter<'_, InsnId> { self.params.iter() } /// Return an iterator over insns - pub fn insns(&self) -> Iter<InsnId> { + pub fn insns(&self) -> Iter<'_, InsnId> { self.insns.iter() } } @@ -559,7 +1649,7 @@ impl Block { /// Pretty printer for [`Function`]. pub struct FunctionPrinter<'a> { fun: &'a Function, - display_snapshot: bool, + display_snapshot_and_tp_patchpoints: bool, ptr_map: PtrPrintMap, } @@ -569,16 +1659,32 @@ impl<'a> FunctionPrinter<'a> { if cfg!(test) { ptr_map.map_ptrs = true; } - Self { fun, display_snapshot: false, ptr_map } + Self { fun, display_snapshot_and_tp_patchpoints: false, ptr_map } } pub fn with_snapshot(fun: &'a Function) -> FunctionPrinter<'a> { let mut printer = Self::without_snapshot(fun); - printer.display_snapshot = true; + printer.display_snapshot_and_tp_patchpoints = true; printer } } +/// Pretty printer for [`Function`]. +pub struct FunctionGraphvizPrinter<'a> { + fun: &'a Function, + ptr_map: PtrPrintMap, +} + +impl<'a> FunctionGraphvizPrinter<'a> { + pub fn new(fun: &'a Function) -> Self { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + Self { fun, ptr_map } + } +} + /// Union-Find (Disjoint-Set) is a data structure for managing disjoint sets that has an interface /// of two operations: /// @@ -618,7 +1724,7 @@ impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { /// Private. Return the internal representation of the forwarding pointer for a given element. fn at(&self, idx: T) -> Option<T> { - self.forwarded.get(idx.into()).map(|x| *x).flatten() + self.forwarded.get(idx.into()).copied().flatten() } /// Private. Set the internal representation of the forwarding pointer for the given element @@ -669,25 +1775,185 @@ impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { } } +#[derive(Clone, Debug, PartialEq)] +pub enum ValidationError { + BlockHasNoTerminator(BlockId), + // The terminator and its actual position + TerminatorNotAtEnd(BlockId, InsnId, usize), + /// Expected length, actual length + MismatchedBlockArity(BlockId, usize, usize), + JumpTargetNotInRPO(BlockId), + // The offending instruction, and the operand + OperandNotDefined(BlockId, InsnId, InsnId), + /// The offending block and instruction + DuplicateInstruction(BlockId, InsnId), + /// The offending instruction, its operand, expected type string, actual type string + MismatchedOperandType(InsnId, InsnId, String, String), + MiscValidationError(InsnId, String), +} + +fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t, send_insn: InsnId, args: &[InsnId]) -> bool { + let mut can_send = true; + let mut count_failure = |counter| { + can_send = false; + function.push_insn(block, Insn::IncrCounter(counter)); + }; + let params = unsafe { iseq.params() }; + + use Counter::*; + if 0 != params.flags.has_rest() { count_failure(complex_arg_pass_param_rest) } + if 0 != params.flags.has_post() { count_failure(complex_arg_pass_param_post) } + if 0 != params.flags.has_block() { count_failure(complex_arg_pass_param_block) } + if 0 != params.flags.forwardable() { count_failure(complex_arg_pass_param_forwardable) } + + if 0 != params.flags.has_kwrest() { count_failure(complex_arg_pass_param_kwrest) } + if 0 != params.flags.has_kw() { + let keyword = params.keyword; + if !keyword.is_null() { + let num = unsafe { (*keyword).num }; + let required_num = unsafe { (*keyword).required_num }; + // Only support required keywords for now (no optional keywords) + if num != required_num { + count_failure(complex_arg_pass_param_kw_opt) + } + } + } + + if !can_send { + function.set_dynamic_send_reason(send_insn, ComplexArgPass); + return false; + } + + // asm.ccall() doesn't support 6+ args + if args.len() + 1 > C_ARG_OPNDS.len() { // +1 for self + function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); + return false; + } + + // Because we exclude e.g. post parameters above, they are also excluded from the sum below. + let lead_num = params.lead_num; + let opt_num = params.opt_num; + let keyword = params.keyword; + let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } }; + let req_num = lead_num + kw_req_num; + can_send = c_int::try_from(args.len()) + .as_ref() + .map(|argc| (req_num..=req_num + opt_num).contains(argc)) + .unwrap_or(false); + if !can_send { + function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); + return false + } + + can_send +} + /// A [`Function`], which is analogous to a Ruby ISeq, is a control-flow graph of [`Block`]s /// containing instructions. #[derive(Debug)] pub struct Function { // ISEQ this function refers to iseq: *const rb_iseq_t, - // The types for the parameters of this function + /// The types for the parameters of this function. They are copied to the type + /// of entry block params after infer_types() fills Empty to all insn_types. param_types: Vec<Type>, - // TODO: get method name and source location from the ISEQ - insns: Vec<Insn>, union_find: std::cell::RefCell<UnionFind<InsnId>>, insn_types: Vec<Type>, blocks: Vec<Block>, + /// Entry block for the interpreter entry_block: BlockId, + /// Entry block for JIT-to-JIT calls. Length will be `opt_num+1`, for callers + /// fulfilling `(0..=opt_num)` optional parameters. + jit_entry_blocks: Vec<BlockId>, profiles: Option<ProfileOracle>, } +/// The kind of a value an ISEQ returns +enum IseqReturn { + Value(VALUE), + LocalVariable(u32), + Receiver, + // Builtin descriptor and return type (if known) + InvokeLeafBuiltin(rb_builtin_function, Option<Type>), +} + +unsafe extern "C" { + fn rb_simple_iseq_p(iseq: IseqPtr) -> bool; +} + +/// Return the ISEQ's return value if it consists of one simple instruction and leave. +fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<InsnId>, ci_flags: u32) -> Option<IseqReturn> { + // Expect only two instructions and one possible operand + // NOTE: If an ISEQ has an optional keyword parameter with a default value that requires + // computation, the ISEQ will always have more than two instructions and won't be inlined. + + // Get the first two instructions + let first_insn = iseq_opcode_at_idx(iseq, 0); + let second_insn = iseq_opcode_at_idx(iseq, insn_len(first_insn as usize)); + + // Extract the return value if known + if second_insn != YARVINSN_leave { + return None; + } + match first_insn { + YARVINSN_getlocal_WC_0 => { + // Accept only cases where only positional arguments are used by both the callee and the caller. + // Keyword arguments may be specified by the callee or the caller but not used. + if captured_opnd.is_some() + // Equivalent to `VM_CALL_ARGS_SIMPLE - VM_CALL_KWARG - has_block_iseq` + || ci_flags & ( + VM_CALL_ARGS_SPLAT + | VM_CALL_KW_SPLAT + | VM_CALL_ARGS_BLOCKARG + | VM_CALL_FORWARDING + ) != 0 + { + return None; + } + + let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); + let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + + // Only inline if the local is a parameter (not a method-defined local) as we are indexing args. + let param_size = unsafe { rb_get_iseq_body_param_size(iseq) } as usize; + if local_idx >= param_size { + return None; + } + + if unsafe { rb_simple_iseq_p(iseq) } { + return Some(IseqReturn::LocalVariable(local_idx.try_into().unwrap())); + } + + // TODO(max): Support only_kwparam case where the local_idx is a positional parameter + + None + } + YARVINSN_putnil => Some(IseqReturn::Value(Qnil)), + YARVINSN_putobject => Some(IseqReturn::Value(unsafe { *rb_iseq_pc_at_idx(iseq, 1) })), + YARVINSN_putobject_INT2FIX_0_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(0))), + YARVINSN_putobject_INT2FIX_1_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(1))), + // We don't support invokeblock for now. Such ISEQs are likely not used by blocks anyway. + YARVINSN_putself if captured_opnd.is_none() => Some(IseqReturn::Receiver), + YARVINSN_opt_invokebuiltin_delegate_leave => { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let argc = bf.argc as usize; + if argc != 0 { return None; } + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + if !leaf { return None; } + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + Some(IseqReturn::InvokeLeafBuiltin(bf, return_type)) + } + _ => None, + } +} + impl Function { fn new(iseq: *const rb_iseq_t) -> Function { Function { @@ -697,6 +1963,7 @@ impl Function { union_find: UnionFind::new().into(), blocks: vec![Block::default()], entry_block: BlockId(0), + jit_entry_blocks: vec![], param_types: vec![], profiles: None, } @@ -715,8 +1982,8 @@ impl Function { } // Add an instruction to an SSA block - fn push_insn(&mut self, block: BlockId, insn: Insn) -> InsnId { - let is_param = matches!(insn, Insn::Param { .. }); + pub fn push_insn(&mut self, block: BlockId, insn: Insn) -> InsnId { + let is_param = matches!(insn, Insn::Param); let id = self.new_insn(insn); if is_param { self.blocks[block.0].params.push(id); @@ -745,27 +2012,71 @@ impl Function { } } - fn new_block(&mut self) -> BlockId { + fn new_block(&mut self, insn_idx: u32) -> BlockId { let id = BlockId(self.blocks.len()); - self.blocks.push(Block::default()); + let block = Block { + insn_idx, + .. Block::default() + }; + self.blocks.push(block); id } + fn remove_block(&mut self, block_id: BlockId) { + if BlockId(self.blocks.len() - 1) != block_id { + panic!("Can only remove the last block"); + } + self.blocks.pop(); + } + /// Return a reference to the Block at the given index. pub fn block(&self, block_id: BlockId) -> &Block { &self.blocks[block_id.0] } + /// Return a reference to the entry block. + pub fn entry_block(&self) -> &Block { + &self.blocks[self.entry_block.0] + } + /// Return the number of blocks pub fn num_blocks(&self) -> usize { self.blocks.len() } + pub fn assume_single_ractor_mode(&mut self, block: BlockId, state: InsnId) -> bool { + if unsafe { rb_jit_multi_ractor_p() } { + false + } else { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::SingleRactorMode, state }); + true + } + } + + /// Assume that objects of a given class will have no singleton class. + /// Returns true if safe to assume so and emits a PatchPoint. + /// Returns false if we've already seen a singleton class for this class, + /// to avoid an invalidation loop. + pub fn assume_no_singleton_classes(&mut self, block: BlockId, klass: VALUE, state: InsnId) -> bool { + if !klass.instance_can_have_singleton_class() { + // This class can never have a singleton class, so no patchpoint needed. + return true; + } + if has_singleton_class_of(klass) { + // We've seen a singleton class for this klass. Disable the optimization + // to avoid an invalidation loop. + return false; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass }, state }); + true + } + /// Return a copy of the instruction where the instruction and its operands have been read from /// the union-find table (to find the current most-optimized version of this instruction). See /// [`UnionFind`] for more. /// - /// Use for pattern matching over instructions in a union-find-safe way. For example: + /// This is _the_ function for reading [`Insn`]. Use frequently. Example: + /// /// ```rust /// match func.find(insn_id) { /// IfTrue { val, target } if func.is_truthy(val) => { @@ -805,94 +2116,248 @@ impl Function { let insn_id = find!(insn_id); use Insn::*; match &self.insns[insn_id.0] { - result@(PutSelf | Const {..} | Param {..} | GetConstantPath {..} - | PatchPoint {..}) => result.clone(), - Snapshot { state: FrameState { iseq, insn_idx, pc, stack, locals } } => + result@(Const {..} + | Param + | GetConstantPath {..} + | IsBlockGiven + | PatchPoint {..} + | PutSpecialObject {..} + | GetGlobal {..} + | GetLocal {..} + | SideExit {..} + | EntryPoint {..} + | LoadPC + | LoadEC + | LoadSelf + | IncrCounterPtr {..} + | IncrCounter(_)) => result.clone(), + &Snapshot { state: FrameState { iseq, insn_idx, pc, ref stack, ref locals } } => Snapshot { state: FrameState { - iseq: *iseq, - insn_idx: *insn_idx, - pc: *pc, - stack: stack.iter().map(|v| find!(*v)).collect(), - locals: locals.iter().map(|v| find!(*v)).collect(), + iseq, + insn_idx, + pc, + stack: find_vec!(stack), + locals: find_vec!(locals), } }, - Return { val } => Return { val: find!(*val) }, - StringCopy { val } => StringCopy { val: find!(*val) }, - StringIntern { val } => StringIntern { val: find!(*val) }, - Test { val } => Test { val: find!(*val) }, + &Return { val } => Return { val: find!(val) }, + &FixnumBitCheck { val, index } => FixnumBitCheck { val: find!(val), index }, + &Throw { throw_state, val, state } => Throw { throw_state, val: find!(val), state }, + &StringCopy { val, chilled, state } => StringCopy { val: find!(val), chilled, state }, + &StringIntern { val, state } => StringIntern { val: find!(val), state: find!(state) }, + &StringConcat { ref strings, state } => StringConcat { strings: find_vec!(strings), state: find!(state) }, + &StringGetbyte { string, index } => StringGetbyte { string: find!(string), index: find!(index) }, + &StringSetbyteFixnum { string, index, value } => StringSetbyteFixnum { string: find!(string), index: find!(index), value: find!(value) }, + &StringAppend { recv, other, state } => StringAppend { recv: find!(recv), other: find!(other), state: find!(state) }, + &StringAppendCodepoint { recv, other, state } => StringAppendCodepoint { recv: find!(recv), other: find!(other), state: find!(state) }, + &ToRegexp { opt, ref values, state } => ToRegexp { opt, values: find_vec!(values), state }, + &Test { val } => Test { val: find!(val) }, + &IsNil { val } => IsNil { val: find!(val) }, + &IsMethodCfunc { val, cd, cfunc, state } => IsMethodCfunc { val: find!(val), cd, cfunc, state }, + &IsBitEqual { left, right } => IsBitEqual { left: find!(left), right: find!(right) }, + &IsBitNotEqual { left, right } => IsBitNotEqual { left: find!(left), right: find!(right) }, + &BoxBool { val } => BoxBool { val: find!(val) }, + &BoxFixnum { val, state } => BoxFixnum { val: find!(val), state: find!(state) }, + &UnboxFixnum { val } => UnboxFixnum { val: find!(val) }, + &FixnumAref { recv, index } => FixnumAref { recv: find!(recv), index: find!(index) }, Jump(target) => Jump(find_branch_edge!(target)), - IfTrue { val, target } => IfTrue { val: find!(*val), target: find_branch_edge!(target) }, - IfFalse { val, target } => IfFalse { val: find!(*val), target: find_branch_edge!(target) }, - GuardType { val, guard_type, state } => GuardType { val: find!(*val), guard_type: *guard_type, state: *state }, - GuardBitEquals { val, expected, state } => GuardBitEquals { val: find!(*val), expected: *expected, state: *state }, - FixnumAdd { left, right, state } => FixnumAdd { left: find!(*left), right: find!(*right), state: *state }, - FixnumSub { left, right, state } => FixnumSub { left: find!(*left), right: find!(*right), state: *state }, - FixnumMult { left, right, state } => FixnumMult { left: find!(*left), right: find!(*right), state: *state }, - FixnumDiv { left, right, state } => FixnumDiv { left: find!(*left), right: find!(*right), state: *state }, - FixnumMod { left, right, state } => FixnumMod { left: find!(*left), right: find!(*right), state: *state }, - FixnumNeq { left, right } => FixnumNeq { left: find!(*left), right: find!(*right) }, - FixnumEq { left, right } => FixnumEq { left: find!(*left), right: find!(*right) }, - FixnumGt { left, right } => FixnumGt { left: find!(*left), right: find!(*right) }, - FixnumGe { left, right } => FixnumGe { left: find!(*left), right: find!(*right) }, - FixnumLt { left, right } => FixnumLt { left: find!(*left), right: find!(*right) }, - FixnumLe { left, right } => FixnumLe { left: find!(*left), right: find!(*right) }, - SendWithoutBlock { self_val, call_info, cd, args, state } => SendWithoutBlock { - self_val: find!(*self_val), - call_info: call_info.clone(), - cd: *cd, - args: args.iter().map(|arg| find!(*arg)).collect(), - state: *state, + &IfTrue { val, ref target } => IfTrue { val: find!(val), target: find_branch_edge!(target) }, + &IfFalse { val, ref target } => IfFalse { val: find!(val), target: find_branch_edge!(target) }, + &GuardType { val, guard_type, state } => GuardType { val: find!(val), guard_type, state }, + &GuardTypeNot { val, guard_type, state } => GuardTypeNot { val: find!(val), guard_type, state }, + &GuardBitEquals { val, expected, reason, state } => GuardBitEquals { val: find!(val), expected, reason, state }, + &GuardShape { val, shape, state } => GuardShape { val: find!(val), shape, state }, + &GuardBlockParamProxy { level, state } => GuardBlockParamProxy { level, state: find!(state) }, + &GuardNotFrozen { recv, state } => GuardNotFrozen { recv: find!(recv), state }, + &GuardNotShared { recv, state } => GuardNotShared { recv: find!(recv), state }, + &GuardGreaterEq { left, right, state } => GuardGreaterEq { left: find!(left), right: find!(right), state }, + &GuardLess { left, right, state } => GuardLess { left: find!(left), right: find!(right), state }, + &GuardSuperMethodEntry { cme, state } => GuardSuperMethodEntry { cme, state }, + &GetBlockHandler => GetBlockHandler, + &FixnumAdd { left, right, state } => FixnumAdd { left: find!(left), right: find!(right), state }, + &FixnumSub { left, right, state } => FixnumSub { left: find!(left), right: find!(right), state }, + &FixnumMult { left, right, state } => FixnumMult { left: find!(left), right: find!(right), state }, + &FixnumDiv { left, right, state } => FixnumDiv { left: find!(left), right: find!(right), state }, + &FixnumMod { left, right, state } => FixnumMod { left: find!(left), right: find!(right), state }, + &FixnumNeq { left, right } => FixnumNeq { left: find!(left), right: find!(right) }, + &FixnumEq { left, right } => FixnumEq { left: find!(left), right: find!(right) }, + &FixnumGt { left, right } => FixnumGt { left: find!(left), right: find!(right) }, + &FixnumGe { left, right } => FixnumGe { left: find!(left), right: find!(right) }, + &FixnumLt { left, right } => FixnumLt { left: find!(left), right: find!(right) }, + &FixnumLe { left, right } => FixnumLe { left: find!(left), right: find!(right) }, + &FixnumAnd { left, right } => FixnumAnd { left: find!(left), right: find!(right) }, + &FixnumOr { left, right } => FixnumOr { left: find!(left), right: find!(right) }, + &FixnumXor { left, right } => FixnumXor { left: find!(left), right: find!(right) }, + &FixnumLShift { left, right, state } => FixnumLShift { left: find!(left), right: find!(right), state }, + &FixnumRShift { left, right } => FixnumRShift { left: find!(left), right: find!(right) }, + &ObjToString { val, cd, state } => ObjToString { + val: find!(val), + cd, + state, + }, + &AnyToString { val, str, state } => AnyToString { + val: find!(val), + str: find!(str), + state, + }, + &SendWithoutBlock { recv, cd, ref args, state, reason } => SendWithoutBlock { + recv: find!(recv), + cd, + args: find_vec!(args), + state, + reason, + }, + &SendWithoutBlockDirect { recv, cd, cme, iseq, ref args, state } => SendWithoutBlockDirect { + recv: find!(recv), + cd, + cme, + iseq, + args: find_vec!(args), + state, + }, + &Send { recv, cd, blockiseq, ref args, state, reason } => Send { + recv: find!(recv), + cd, + blockiseq, + args: find_vec!(args), + state, + reason, }, - SendWithoutBlockDirect { self_val, call_info, cd, iseq, args, state } => SendWithoutBlockDirect { - self_val: find!(*self_val), - call_info: call_info.clone(), - cd: *cd, - iseq: *iseq, - args: args.iter().map(|arg| find!(*arg)).collect(), - state: *state, + &SendForward { recv, cd, blockiseq, ref args, state, reason } => SendForward { + recv: find!(recv), + cd, + blockiseq, + args: find_vec!(args), + state, + reason, }, - Send { self_val, call_info, cd, blockiseq, args, state } => Send { - self_val: find!(*self_val), - call_info: call_info.clone(), - cd: *cd, - blockiseq: *blockiseq, - args: args.iter().map(|arg| find!(*arg)).collect(), - state: *state, + &InvokeSuper { recv, cd, blockiseq, ref args, state, reason } => InvokeSuper { + recv: find!(recv), + cd, + blockiseq, + args: find_vec!(args), + state, + reason, }, - ArraySet { array, idx, val } => ArraySet { array: find!(*array), idx: *idx, val: find!(*val) }, - ArrayDup { val , state } => ArrayDup { val: find!(*val), state: *state }, - &CCall { cfun, ref args, name, return_type, elidable } => CCall { cfun: cfun, args: args.iter().map(|arg| find!(*arg)).collect(), name: name, return_type: return_type, elidable }, - Defined { .. } => todo!("find(Defined)"), - NewArray { elements, state } => NewArray { elements: find_vec!(*elements), state: find!(*state) }, - ArrayMax { elements, state } => ArrayMax { elements: find_vec!(*elements), state: find!(*state) }, + &InvokeBlock { cd, ref args, state, reason } => InvokeBlock { + cd, + args: find_vec!(args), + state, + reason, + }, + &InvokeProc { recv, ref args, state, kw_splat } => InvokeProc { + recv: find!(recv), + args: find_vec!(args), + state: find!(state), + kw_splat, + }, + &InvokeBuiltin { bf, recv, ref args, state, leaf, return_type } => InvokeBuiltin { bf, recv: find!(recv), args: find_vec!(args), state, leaf, return_type }, + &ArrayDup { val, state } => ArrayDup { val: find!(val), state }, + &HashDup { val, state } => HashDup { val: find!(val), state }, + &HashAref { hash, key, state } => HashAref { hash: find!(hash), key: find!(key), state }, + &HashAset { hash, key, val, state } => HashAset { hash: find!(hash), key: find!(key), val: find!(val), state }, + &ObjectAlloc { val, state } => ObjectAlloc { val: find!(val), state }, + &ObjectAllocClass { class, state } => ObjectAllocClass { class, state: find!(state) }, + &CCall { cfunc, recv, ref args, name, return_type, elidable } => CCall { cfunc, recv: find!(recv), args: find_vec!(args), name, return_type, elidable }, + &CCallWithFrame { cd, cfunc, recv, ref args, cme, name, state, return_type, elidable, blockiseq } => CCallWithFrame { + cd, + cfunc, + recv: find!(recv), + args: find_vec!(args), + cme, + name, + state: find!(state), + return_type, + elidable, + blockiseq, + }, + &CCallVariadic { cfunc, recv, ref args, cme, name, state, return_type, elidable, blockiseq } => CCallVariadic { + cfunc, recv: find!(recv), args: find_vec!(args), cme, name, state, return_type, elidable, blockiseq + }, + &Defined { op_type, obj, pushval, v, state } => Defined { op_type, obj, pushval, v: find!(v), state: find!(state) }, + &DefinedIvar { self_val, pushval, id, state } => DefinedIvar { self_val: find!(self_val), pushval, id, state }, + &NewArray { ref elements, state } => NewArray { elements: find_vec!(elements), state: find!(state) }, + &NewHash { ref elements, state } => NewHash { elements: find_vec!(elements), state: find!(state) }, + &NewRange { low, high, flag, state } => NewRange { low: find!(low), high: find!(high), flag, state: find!(state) }, + &NewRangeFixnum { low, high, flag, state } => NewRangeFixnum { low: find!(low), high: find!(high), flag, state: find!(state) }, + &ArrayAref { array, index } => ArrayAref { array: find!(array), index: find!(index) }, + &ArrayAset { array, index, val } => ArrayAset { array: find!(array), index: find!(index), val: find!(val) }, + &ArrayPop { array, state } => ArrayPop { array: find!(array), state: find!(state) }, + &ArrayLength { array } => ArrayLength { array: find!(array) }, + &ArrayMax { ref elements, state } => ArrayMax { elements: find_vec!(elements), state: find!(state) }, + &ArrayInclude { ref elements, target, state } => ArrayInclude { elements: find_vec!(elements), target: find!(target), state: find!(state) }, + &ArrayPackBuffer { ref elements, fmt, buffer, state } => ArrayPackBuffer { elements: find_vec!(elements), fmt: find!(fmt), buffer: find!(buffer), state: find!(state) }, + &DupArrayInclude { ary, target, state } => DupArrayInclude { ary, target: find!(target), state: find!(state) }, + &ArrayHash { ref elements, state } => ArrayHash { elements: find_vec!(elements), state }, + &SetGlobal { id, val, state } => SetGlobal { id, val: find!(val), state }, + &GetIvar { self_val, id, ic, state } => GetIvar { self_val: find!(self_val), id, ic, state }, + &LoadField { recv, id, offset, return_type } => LoadField { recv: find!(recv), id, offset, return_type }, + &StoreField { recv, id, offset, val } => StoreField { recv: find!(recv), id, offset, val: find!(val) }, + &WriteBarrier { recv, val } => WriteBarrier { recv: find!(recv), val: find!(val) }, + &SetIvar { self_val, id, ic, val, state } => SetIvar { self_val: find!(self_val), id, ic, val: find!(val), state }, + &GetClassVar { id, ic, state } => GetClassVar { id, ic, state }, + &SetClassVar { id, val, ic, state } => SetClassVar { id, val: find!(val), ic, state }, + &SetLocal { val, ep_offset, level } => SetLocal { val: find!(val), ep_offset, level }, + &GetSpecialSymbol { symbol_type, state } => GetSpecialSymbol { symbol_type, state }, + &GetSpecialNumber { nth, state } => GetSpecialNumber { nth, state }, + &ToArray { val, state } => ToArray { val: find!(val), state }, + &ToNewArray { val, state } => ToNewArray { val: find!(val), state }, + &ArrayExtend { left, right, state } => ArrayExtend { left: find!(left), right: find!(right), state }, + &ArrayPush { array, val, state } => ArrayPush { array: find!(array), val: find!(val), state }, + &CheckInterrupts { state } => CheckInterrupts { state }, + &IsA { val, class } => IsA { val: find!(val), class: find!(class) }, + } + } + + /// Update DynamicSendReason for the instruction at insn_id + fn set_dynamic_send_reason(&mut self, insn_id: InsnId, dynamic_send_reason: SendFallbackReason) { + use Insn::*; + if get_option!(stats) || get_option!(dump_hir_opt).is_some() || cfg!(test) { + match self.insns.get_mut(insn_id.0).unwrap() { + Send { reason, .. } + | SendForward { reason, .. } + | SendWithoutBlock { reason, .. } + | InvokeSuper { reason, .. } + | InvokeBlock { reason, .. } + => *reason = dynamic_send_reason, + _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id)) + } } } /// Replace `insn` with the new instruction `replacement`, which will get appended to `insns`. fn make_equal_to(&mut self, insn: InsnId, replacement: InsnId) { + assert!(self.insns[insn.0].has_output(), + "Don't use make_equal_to for instruction with no output"); + assert!(self.insns[replacement.0].has_output(), + "Can't replace instruction that has output with instruction that has no output"); // Don't push it to the block self.union_find.borrow_mut().make_equal_to(insn, replacement); } - fn type_of(&self, insn: InsnId) -> Type { + pub fn type_of(&self, insn: InsnId) -> Type { assert!(self.insns[insn.0].has_output()); self.insn_types[self.union_find.borrow_mut().find(insn).0] } /// Check if the type of `insn` is a subtype of `ty`. - fn is_a(&self, insn: InsnId, ty: Type) -> bool { + pub fn is_a(&self, insn: InsnId, ty: Type) -> bool { self.type_of(insn).is_subtype(ty) } fn infer_type(&self, insn: InsnId) -> Type { assert!(self.insns[insn.0].has_output()); match &self.insns[insn.0] { - Insn::Param { .. } => unimplemented!("params should not be present in block.insns"), - Insn::ArraySet { .. } | Insn::Snapshot { .. } | Insn::Jump(_) - | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::Return { .. } - | Insn::PatchPoint { .. } => - panic!("Cannot infer type of instruction with no output"), + Insn::Param => unimplemented!("params should not be present in block.insns"), + Insn::SetGlobal { .. } | Insn::Jump(_) | Insn::EntryPoint { .. } + | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::Return { .. } | Insn::Throw { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } + | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetLocal { .. } + | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::GuardBlockParamProxy { .. } | Insn::GuardSuperMethodEntry { .. } + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } | Insn::ArrayAset { .. } => + panic!("Cannot infer type of instruction with no output: {}. See Insn::has_output().", self.insns[insn.0]), Insn::Const { val: Const::Value(val) } => Type::from_value(*val), Insn::Const { val: Const::CBool(val) } => Type::from_cbool(*val), Insn::Const { val: Const::CInt8(val) } => Type::from_cint(types::CInt8, *val as i64), @@ -902,19 +2367,56 @@ impl Function { Insn::Const { val: Const::CUInt8(val) } => Type::from_cint(types::CUInt8, *val as i64), Insn::Const { val: Const::CUInt16(val) } => Type::from_cint(types::CUInt16, *val as i64), Insn::Const { val: Const::CUInt32(val) } => Type::from_cint(types::CUInt32, *val as i64), + Insn::Const { val: Const::CShape(val) } => Type::from_cint(types::CShape, val.0 as i64), Insn::Const { val: Const::CUInt64(val) } => Type::from_cint(types::CUInt64, *val as i64), - Insn::Const { val: Const::CPtr(val) } => Type::from_cint(types::CPtr, *val as i64), + Insn::Const { val: Const::CPtr(val) } => Type::from_cptr(*val), Insn::Const { val: Const::CDouble(val) } => Type::from_double(*val), Insn::Test { val } if self.type_of(*val).is_known_falsy() => Type::from_cbool(false), Insn::Test { val } if self.type_of(*val).is_known_truthy() => Type::from_cbool(true), Insn::Test { .. } => types::CBool, + Insn::IsNil { val } if self.is_a(*val, types::NilClass) => Type::from_cbool(true), + Insn::IsNil { val } if !self.type_of(*val).could_be(types::NilClass) => Type::from_cbool(false), + Insn::IsNil { .. } => types::CBool, + Insn::IsMethodCfunc { .. } => types::CBool, + Insn::IsBitEqual { .. } => types::CBool, + Insn::IsBitNotEqual { .. } => types::CBool, + Insn::BoxBool { .. } => types::BoolExact, + Insn::BoxFixnum { .. } => types::Fixnum, + Insn::UnboxFixnum { val } => self + .type_of(*val) + .fixnum_value() + .map_or(types::CInt64, |fixnum| Type::from_cint(types::CInt64, fixnum)), + Insn::FixnumAref { .. } => types::Fixnum, Insn::StringCopy { .. } => types::StringExact, - Insn::StringIntern { .. } => types::StringExact, + Insn::StringIntern { .. } => types::Symbol, + Insn::StringConcat { .. } => types::StringExact, + Insn::StringGetbyte { .. } => types::Fixnum, + Insn::StringSetbyteFixnum { .. } => types::Fixnum, + Insn::StringAppend { .. } => types::StringExact, + Insn::StringAppendCodepoint { .. } => types::StringExact, + Insn::ToRegexp { .. } => types::RegexpExact, Insn::NewArray { .. } => types::ArrayExact, Insn::ArrayDup { .. } => types::ArrayExact, + Insn::ArrayAref { .. } => types::BasicObject, + Insn::ArrayPop { .. } => types::BasicObject, + Insn::ArrayLength { .. } => types::CInt64, + Insn::HashAref { .. } => types::BasicObject, + Insn::NewHash { .. } => types::HashExact, + Insn::HashDup { .. } => types::HashExact, + Insn::NewRange { .. } => types::RangeExact, + Insn::NewRangeFixnum { .. } => types::RangeExact, + Insn::ObjectAlloc { .. } => types::HeapBasicObject, + Insn::ObjectAllocClass { class, .. } => Type::from_class(*class), + &Insn::CCallWithFrame { return_type, .. } => return_type, Insn::CCall { return_type, .. } => *return_type, + &Insn::CCallVariadic { return_type, .. } => return_type, Insn::GuardType { val, guard_type, .. } => self.type_of(*val).intersection(*guard_type), - Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_value(*expected)), + Insn::GuardTypeNot { .. } => types::BasicObject, + Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_const(*expected)), + Insn::GuardShape { val, .. } => self.type_of(*val), + Insn::GuardNotFrozen { recv, .. } | Insn::GuardNotShared { recv, .. } => self.type_of(*recv), + Insn::GuardLess { left, .. } => self.type_of(*left), + Insn::GuardGreaterEq { left, .. } => self.type_of(*left), Insn::FixnumAdd { .. } => types::Fixnum, Insn::FixnumSub { .. } => types::Fixnum, Insn::FixnumMult { .. } => types::Fixnum, @@ -926,13 +2428,84 @@ impl Function { Insn::FixnumLe { .. } => types::BoolExact, Insn::FixnumGt { .. } => types::BoolExact, Insn::FixnumGe { .. } => types::BoolExact, + Insn::FixnumAnd { .. } => types::Fixnum, + Insn::FixnumOr { .. } => types::Fixnum, + Insn::FixnumXor { .. } => types::Fixnum, + Insn::FixnumLShift { .. } => types::Fixnum, + Insn::FixnumRShift { .. } => types::Fixnum, + Insn::PutSpecialObject { .. } => types::BasicObject, Insn::SendWithoutBlock { .. } => types::BasicObject, Insn::SendWithoutBlockDirect { .. } => types::BasicObject, Insn::Send { .. } => types::BasicObject, - Insn::PutSelf => types::BasicObject, - Insn::Defined { .. } => types::BasicObject, + Insn::SendForward { .. } => types::BasicObject, + Insn::InvokeSuper { .. } => types::BasicObject, + Insn::InvokeBlock { .. } => types::BasicObject, + Insn::InvokeProc { .. } => types::BasicObject, + Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), + Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), + Insn::DefinedIvar { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), Insn::GetConstantPath { .. } => types::BasicObject, + Insn::IsBlockGiven => types::BoolExact, + Insn::FixnumBitCheck { .. } => types::BoolExact, Insn::ArrayMax { .. } => types::BasicObject, + Insn::ArrayInclude { .. } => types::BoolExact, + Insn::ArrayPackBuffer { .. } => types::String, + Insn::DupArrayInclude { .. } => types::BoolExact, + Insn::ArrayHash { .. } => types::Fixnum, + Insn::GetGlobal { .. } => types::BasicObject, + Insn::GetIvar { .. } => types::BasicObject, + Insn::LoadPC => types::CPtr, + Insn::LoadEC => types::CPtr, + Insn::LoadSelf => types::BasicObject, + &Insn::LoadField { return_type, .. } => return_type, + Insn::GetSpecialSymbol { .. } => types::BasicObject, + Insn::GetSpecialNumber { .. } => types::BasicObject, + Insn::GetClassVar { .. } => types::BasicObject, + Insn::ToNewArray { .. } => types::ArrayExact, + Insn::ToArray { .. } => types::ArrayExact, + Insn::ObjToString { .. } => types::BasicObject, + Insn::AnyToString { .. } => types::String, + Insn::GetLocal { rest_param: true, .. } => types::ArrayExact, + Insn::GetLocal { .. } => types::BasicObject, + Insn::GetBlockHandler => types::RubyValue, + // The type of Snapshot doesn't really matter; it's never materialized. It's used only + // as a reference for FrameState, which we use to generate side-exit code. + Insn::Snapshot { .. } => types::Any, + Insn::IsA { .. } => types::BoolExact, + } + } + + /// Set self.param_types. They are copied to the param types of jit_entry_blocks. + fn set_param_types(&mut self) { + let iseq = self.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + self.param_types.push(types::BasicObject); // self + for local_idx in 0..param_size { + let param_type = if Some(local_idx as i32) == rest_param_idx { + types::ArrayExact // Rest parameters are always ArrayExact + } else { + types::BasicObject + }; + self.param_types.push(param_type); + } + } + + /// Copy self.param_types to the param types of jit_entry_blocks. + fn copy_param_types(&mut self) { + for jit_entry_block in self.jit_entry_blocks.iter() { + let entry_params = self.blocks[jit_entry_block.0].params.iter(); + let param_types = self.param_types.iter(); + assert!( + param_types.len() >= entry_params.len(), + "param types should be initialized before type inference", + ); + for (param, param_type) in std::iter::zip(entry_params, param_types) { + // We know that function parameters are BasicObject or some subclass + self.insn_types[param.0] = *param_type; + } } } @@ -940,33 +2513,26 @@ impl Function { // Reset all types self.insn_types.fill(types::Empty); - // Fill parameter types - let entry_params = self.blocks[self.entry_block.0].params.iter(); - let param_types = self.param_types.iter(); - assert_eq!( - entry_params.len(), - entry_params.len(), - "param types should be initialized before type inference" - ); - for (param, param_type) in std::iter::zip(entry_params, param_types) { - // We know that function parameters are BasicObject or some subclass - self.insn_types[param.0] = *param_type; + // Fill entry parameter types + self.copy_param_types(); + + let mut reachable = BlockSet::with_capacity(self.blocks.len()); + for entry_block in self.entry_blocks() { + reachable.insert(entry_block); } - let rpo = self.rpo(); + // Walk the graph, computing types until fixpoint - let mut reachable = vec![false; self.blocks.len()]; - reachable[self.entry_block.0] = true; + let rpo = self.rpo(); loop { let mut changed = false; - for block in &rpo { - if !reachable[block.0] { continue; } + for &block in &rpo { + if !reachable.get(block) { continue; } for insn_id in &self.blocks[block.0].insns { - let insn = self.find(*insn_id); - let insn_type = match insn { + let insn_type = match self.find(*insn_id) { Insn::IfTrue { val, target: BranchEdge { target, args } } => { assert!(!self.type_of(val).bit_equal(types::Empty)); if self.type_of(val).could_be(Type::from_cbool(true)) { - reachable[target.0] = true; + reachable.insert(target); for (idx, arg) in args.iter().enumerate() { let param = self.blocks[target.0].params[idx]; self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); @@ -977,7 +2543,7 @@ impl Function { Insn::IfFalse { val, target: BranchEdge { target, args } } => { assert!(!self.type_of(val).bit_equal(types::Empty)); if self.type_of(val).could_be(Type::from_cbool(false)) { - reachable[target.0] = true; + reachable.insert(target); for (idx, arg) in args.iter().enumerate() { let param = self.blocks[target.0].params[idx]; self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); @@ -986,14 +2552,14 @@ impl Function { continue; } Insn::Jump(BranchEdge { target, args }) => { - reachable[target.0] = true; + reachable.insert(target); for (idx, arg) in args.iter().enumerate() { let param = self.blocks[target.0].params[idx]; self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); } continue; } - _ if insn.has_output() => self.infer_type(*insn_id), + insn if insn.has_output() => self.infer_type(*insn_id), _ => continue, }; if !self.type_of(*insn_id).bit_equal(insn_type) { @@ -1008,138 +2574,1053 @@ impl Function { } } - /// Return the interpreter-profiled type of the HIR instruction at the given ISEQ instruction - /// index, if it is known. This historical type record is not a guarantee and must be checked - /// with a GuardType or similar. - fn profiled_type_of_at(&self, insn: InsnId, iseq_insn_idx: usize) -> Option<Type> { - let Some(ref profiles) = self.profiles else { return None }; - let Some(entries) = profiles.types.get(&iseq_insn_idx) else { return None }; - for &(entry_insn, entry_type) in entries { - if self.union_find.borrow().find_const(entry_insn) == self.union_find.borrow().find_const(insn) { - return Some(entry_type); + fn chase_insn(&self, insn: InsnId) -> InsnId { + let id = self.union_find.borrow().find_const(insn); + match self.insns[id.0] { + Insn::GuardType { val, .. } + | Insn::GuardTypeNot { val, .. } + | Insn::GuardShape { val, .. } + | Insn::GuardBitEquals { val, .. } => self.chase_insn(val), + _ => id, + } + } + + /// Return the profiled type of the HIR instruction at the given ISEQ instruction + /// index, if it is known to be monomorphic or skewed polymorphic. This historical type + /// record is not a guarantee and must be checked with a GuardType or similar. + fn profiled_type_of_at(&self, insn: InsnId, iseq_insn_idx: usize) -> Option<ProfiledType> { + match self.resolve_receiver_type_from_profile(insn, iseq_insn_idx) { + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => Some(profiled_type), + _ => None, + } + } + + /// Reorder keyword arguments to match the callee's expectation. + /// + /// Returns Ok with reordered arguments if successful, or Err with the fallback reason if not. + fn reorder_keyword_arguments( + &self, + args: &[InsnId], + kwarg: *const rb_callinfo_kwarg, + iseq: IseqPtr, + ) -> Result<Vec<InsnId>, SendFallbackReason> { + let callee_keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if callee_keyword.is_null() { + // Caller is passing kwargs but callee doesn't expect them. + return Err(SendWithoutBlockDirectKeywordMismatch); + } + + let caller_kw_count = unsafe { get_cikw_keyword_len(kwarg) } as usize; + let callee_kw_count = unsafe { (*callee_keyword).num } as usize; + let callee_kw_required = unsafe { (*callee_keyword).required_num } as usize; + let callee_kw_table = unsafe { (*callee_keyword).table }; + + // For now, only handle the case where all keywords are required. + if callee_kw_count != callee_kw_required { + return Err(SendWithoutBlockDirectOptionalKeywords); + } + if caller_kw_count != callee_kw_count { + return Err(SendWithoutBlockDirectKeywordCountMismatch); + } + + // The keyword arguments are the last arguments in the args vector. + let kw_args_start = args.len() - caller_kw_count; + + // Build a mapping from caller keywords to their positions. + let mut caller_kw_order: Vec<ID> = Vec::with_capacity(caller_kw_count); + for i in 0..caller_kw_count { + let sym = unsafe { get_cikw_keywords_idx(kwarg, i as i32) }; + let id = unsafe { rb_sym2id(sym) }; + caller_kw_order.push(id); + } + + // Reorder keyword arguments to match callee expectation. + let mut reordered_kw_args: Vec<InsnId> = Vec::with_capacity(callee_kw_count); + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + + // Find where this keyword is in the caller's order + let mut found = false; + for (j, &caller_id) in caller_kw_order.iter().enumerate() { + if caller_id == expected_id { + reordered_kw_args.push(args[kw_args_start + j]); + found = true; + break; + } + } + + if !found { + // Required keyword not provided by caller which will raise an ArgumentError. + return Err(SendWithoutBlockDirectMissingKeyword); } } - None + + // Replace the keyword arguments with the reordered ones. + let mut processed_args = args[..kw_args_start].to_vec(); + processed_args.extend(reordered_kw_args); + Ok(processed_args) } - fn likely_is_fixnum(&self, val: InsnId, profiled_type: Type) -> bool { - return self.is_a(val, types::Fixnum) || profiled_type.is_subtype(types::Fixnum); + /// Resolve the receiver type for method dispatch optimization. + /// + /// Takes the receiver's Type, receiver HIR instruction, and ISEQ instruction index. + /// First checks if the receiver's class is statically known, otherwise consults profile data. + /// + /// Returns: + /// - `StaticallyKnown` if the receiver's exact class is known at compile-time + /// - Result of [`Self::resolve_receiver_type_from_profile`] if we need to check profile data + fn resolve_receiver_type(&self, recv: InsnId, recv_type: Type, insn_idx: usize) -> ReceiverTypeResolution { + if let Some(class) = recv_type.runtime_exact_ruby_class() { + return ReceiverTypeResolution::StaticallyKnown { class }; + } + self.resolve_receiver_type_from_profile(recv, insn_idx) } - fn coerce_to_fixnum(&mut self, block: BlockId, val: InsnId, state: InsnId) -> InsnId { - if self.is_a(val, types::Fixnum) { return val; } - return self.push_insn(block, Insn::GuardType { val, guard_type: types::Fixnum, state }); + /// Resolve the receiver type for method dispatch optimization from profile data. + /// + /// Returns: + /// - `Monomorphic`/`SkewedPolymorphic` if we have usable profile data + /// - `Polymorphic` if the receiver has multiple types + /// - `Megamorphic`/`SkewedMegamorphic` if the receiver has too many types to optimize + /// (SkewedMegamorphic may be optimized in the future, but for now we don't) + /// - `NoProfile` if we have no type information + fn resolve_receiver_type_from_profile(&self, recv: InsnId, insn_idx: usize) -> ReceiverTypeResolution { + let Some(profiles) = self.profiles.as_ref() else { + return ReceiverTypeResolution::NoProfile; + }; + let Some(entries) = profiles.types.get(&insn_idx) else { + return ReceiverTypeResolution::NoProfile; + }; + let recv = self.chase_insn(recv); + + for (entry_insn, entry_type_summary) in entries { + if self.union_find.borrow().find_const(*entry_insn) == recv { + if entry_type_summary.is_monomorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::Monomorphic { profiled_type }; + } else if entry_type_summary.is_skewed_polymorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedPolymorphic { profiled_type }; + } else if entry_type_summary.is_skewed_megamorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedMegamorphic { profiled_type }; + } else if entry_type_summary.is_polymorphic() { + return ReceiverTypeResolution::Polymorphic; + } else if entry_type_summary.is_megamorphic() { + return ReceiverTypeResolution::Megamorphic; + } + } + } + + ReceiverTypeResolution::NoProfile } - fn arguments_likely_fixnums(&mut self, left: InsnId, right: InsnId, state: InsnId) -> bool { + pub fn assume_expected_cfunc(&mut self, block: BlockId, class: VALUE, method_id: ID, cfunc: *mut c_void, state: InsnId) -> bool { + let cme = unsafe { rb_callable_method_entry(class, method_id) }; + if cme.is_null() { return false; } + let def_type = unsafe { get_cme_def_type(cme) }; + if def_type != VM_METHOD_TYPE_CFUNC { return false; } + if unsafe { get_mct_func(get_cme_def_body_cfunc(cme)) } != cfunc { + return false; + } + self.gen_patch_points_for_optimized_ccall(block, class, method_id, cme, state); + if !self.assume_no_singleton_classes(block, class, state) { + return false; + } + true + } + + pub fn likely_a(&self, val: InsnId, ty: Type, state: InsnId) -> bool { + if self.type_of(val).is_subtype(ty) { + return true; + } let frame_state = self.frame_state(state); - let iseq_insn_idx = frame_state.insn_idx as usize; - let left_profiled_type = self.profiled_type_of_at(left, iseq_insn_idx).unwrap_or(types::BasicObject); - let right_profiled_type = self.profiled_type_of_at(right, iseq_insn_idx).unwrap_or(types::BasicObject); - self.likely_is_fixnum(left, left_profiled_type) && self.likely_is_fixnum(right, right_profiled_type) - } - - fn try_rewrite_fixnum_op(&mut self, block: BlockId, orig_insn_id: InsnId, f: &dyn Fn(InsnId, InsnId) -> Insn, bop: u32, left: InsnId, right: InsnId, state: InsnId) { - if self.arguments_likely_fixnums(left, right, state) { - if bop == BOP_NEQ { - // For opt_neq, the interpreter checks that both neq and eq are unchanged. - self.push_insn(block, Insn::PatchPoint(Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ })); - } - self.push_insn(block, Insn::PatchPoint(Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop })); - let left = self.coerce_to_fixnum(block, left, state); - let right = self.coerce_to_fixnum(block, right, state); - let result = self.push_insn(block, f(left, right)); - self.make_equal_to(orig_insn_id, result); - self.insn_types[result.0] = self.infer_type(result); + let iseq_insn_idx = frame_state.insn_idx; + let Some(profiled_type) = self.profiled_type_of_at(val, iseq_insn_idx) else { + return false; + }; + Type::from_profiled_type(profiled_type).is_subtype(ty) + } + + pub fn coerce_to(&mut self, block: BlockId, val: InsnId, guard_type: Type, state: InsnId) -> InsnId { + if self.is_a(val, guard_type) { return val; } + self.push_insn(block, Insn::GuardType { val, guard_type, state }) + } + + fn count_complex_call_features(&mut self, block: BlockId, ci_flags: c_uint) { + use Counter::*; + if 0 != ci_flags & VM_CALL_ARGS_SPLAT { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_splat)); } + if 0 != ci_flags & VM_CALL_ARGS_BLOCKARG { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_blockarg)); } + if 0 != ci_flags & VM_CALL_KWARG { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_kwarg)); } + if 0 != ci_flags & VM_CALL_KW_SPLAT { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_kw_splat)); } + if 0 != ci_flags & VM_CALL_TAILCALL { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_tailcall)); } + if 0 != ci_flags & VM_CALL_SUPER { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_super)); } + if 0 != ci_flags & VM_CALL_ZSUPER { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_zsuper)); } + if 0 != ci_flags & VM_CALL_FORWARDING { self.push_insn(block, Insn::IncrCounter(complex_arg_pass_caller_forwarding)); } + } + + fn rewrite_if_frozen(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, klass: u32, bop: u32, state: InsnId) { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + // If the basic operation is already redefined, we cannot optimize it. + self.set_dynamic_send_reason(orig_insn_id, SendWithoutBlockBopRedefined); + self.push_insn_id(block, orig_insn_id); + return; + } + let self_type = self.type_of(self_val); + if let Some(obj) = self_type.ruby_object() { + if obj.is_frozen() { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state }); + self.make_equal_to(orig_insn_id, self_val); + return; + } + } + self.push_insn_id(block, orig_insn_id); + } + + fn try_rewrite_freeze(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { + if self.is_a(self_val, types::StringExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else if self.is_a(self_val, types::ArrayExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else if self.is_a(self_val, types::HashExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, HASH_REDEFINED_OP_FLAG, BOP_FREEZE, state); + } else { + self.push_insn_id(block, orig_insn_id); + } + } + + fn try_rewrite_uminus(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { + if self.is_a(self_val, types::StringExact) { + self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_UMINUS, state); } else { self.push_insn_id(block, orig_insn_id); } } + fn is_metaclass(&self, object: VALUE) -> bool { + unsafe { + if RB_TYPE_P(object, RUBY_T_CLASS) && rb_zjit_singleton_class_p(object) { + let attached = rb_class_attached_object(object); + RB_TYPE_P(attached, RUBY_T_CLASS) || RB_TYPE_P(attached, RUBY_T_MODULE) + } else { + false + } + } + } + /// Rewrite SendWithoutBlock opcodes into SendWithoutBlockDirect opcodes if we know the target /// ISEQ statically. This removes run-time method lookups and opens the door for inlining. - fn optimize_direct_sends(&mut self) { + /// Also try and inline constant caches, specialize object allocations, and more. + fn type_specialize(&mut self) { for block in self.rpo() { let old_insns = std::mem::take(&mut self.blocks[block.0].insns); assert!(self.blocks[block.0].insns.is_empty()); for insn_id in old_insns { match self.find(insn_id) { - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "+" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumAdd { left, right, state }, BOP_PLUS, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "-" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumSub { left, right, state }, BOP_MINUS, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "*" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumMult { left, right, state }, BOP_MULT, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "/" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumDiv { left, right, state }, BOP_DIV, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "%" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumMod { left, right, state }, BOP_MOD, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "==" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumEq { left, right }, BOP_EQ, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "!=" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumNeq { left, right }, BOP_NEQ, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "<" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumLt { left, right }, BOP_LT, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == "<=" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumLe { left, right }, BOP_LE, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == ">" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumGt { left, right }, BOP_GT, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, call_info: CallInfo { method_name }, args, state, .. } if method_name == ">=" && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumGe { left, right }, BOP_GE, self_val, args[0], state), - Insn::SendWithoutBlock { mut self_val, call_info, cd, args, state } => { + Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(freeze) && args.is_empty() => + self.try_rewrite_freeze(block, insn_id, recv, state), + Insn::SendWithoutBlock { recv, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => + self.try_rewrite_uminus(block, insn_id, recv, state), + Insn::SendWithoutBlock { mut recv, cd, args, state, .. } => { let frame_state = self.frame_state(state); - let (klass, guard_equal_to) = if let Some(klass) = self.type_of(self_val).runtime_exact_ruby_class() { - // If we know the class statically, use it to fold the lookup at compile-time. - (klass, None) - } else { - // If we know that self is top-self from profile information, guard and use it to fold the lookup at compile-time. - match self.profiled_type_of_at(self_val, frame_state.insn_idx) { - Some(self_type) if self_type.is_top_self() => (self_type.exact_ruby_class().unwrap(), self_type.ruby_object()), - _ => { self.push_insn_id(block, insn_id); continue; } + let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), frame_state.insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } + | ReceiverTypeResolution::Megamorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockMegamorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::Polymorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockPolymorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::NoProfile => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNoProfiles); + } + self.push_insn_id(block, insn_id); + continue; } }; let ci = unsafe { get_call_data_ci(cd) }; // info about the call site + + let flags = unsafe { rb_vm_ci_flag(ci) }; + let mid = unsafe { vm_ci_mid(ci) }; // Do method lookup let mut cme = unsafe { rb_callable_method_entry(klass, mid) }; if cme.is_null() { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::Null)); self.push_insn_id(block, insn_id); continue; } // Load an overloaded cme if applicable. See vm_search_cc(). // It allows you to use a faster ISEQ if possible. cme = unsafe { rb_check_overloaded_cme(cme, ci) }; - let def_type = unsafe { get_cme_def_type(cme) }; - if def_type != VM_METHOD_TYPE_ISEQ { + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedNeedPermission); + self.push_insn_id(block, insn_id); continue; + } + } + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + + // If the call site info indicates that the `Function` has overly complex arguments, then do not optimize into a `SendWithoutBlockDirect`. + // Optimized methods(`VM_METHOD_TYPE_OPTIMIZED`) handle their own argument constraints (e.g., kw_splat for Proc call). + if def_type != VM_METHOD_TYPE_OPTIMIZED && unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + + if def_type == VM_METHOD_TYPE_ISEQ { // TODO(max): Allow non-iseq; cache cme + // Only specialize positional-positional calls + // TODO(max): Handle other kinds of parameter passing + let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; + if !can_direct_send(self, block, iseq, insn_id, args.as_slice()) { + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let (send_state, processed_args) = if !kwarg.is_null() { + match self.reorder_keyword_arguments(&args, kwarg, iseq) { + Ok(reordered) => { + let new_state = self.frame_state(state).with_reordered_args(&reordered); + let snapshot = self.push_insn(block, Insn::Snapshot { state: new_state }); + (snapshot, reordered) + } + Err(reason) => { + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + } + } + } else { + (state, args.clone()) + }; + + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, state: send_state }); + self.make_equal_to(insn_id, send_direct); + } else if def_type == VM_METHOD_TYPE_BMETHOD { + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + // Target ISEQ bmethods. Can't handle for example, `define_method(:foo, &:foo)` + // which makes a `block_type_symbol` bmethod. + if proc_block.type_ != block_type_iseq { + self.set_dynamic_send_reason(insn_id, BmethodNonIseqProc); + self.push_insn_id(block, insn_id); continue; + } + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let iseq = unsafe { *capture.code.iseq.as_ref() }; + + if !can_direct_send(self, block, iseq, insn_id, args.as_slice()) { + self.push_insn_id(block, insn_id); continue; + } + // Can't pass a block to a block for now + assert!((unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_BLOCKARG) == 0, "SendWithoutBlock but has a block arg"); + + // Patch points: + // Check for "defined with an un-shareable Proc in a different Ractor" + if !procv.shareable_p() && !self.assume_single_ractor_mode(block, state) { + // TODO(alan): Turn this into a ractor belonging guard to work better in multi ractor mode. + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let (send_state, processed_args) = if !kwarg.is_null() { + match self.reorder_keyword_arguments(&args, kwarg, iseq) { + Ok(reordered) => { + let new_state = self.frame_state(state).with_reordered_args(&reordered); + let snapshot = self.push_insn(block, Insn::Snapshot { state: new_state }); + (snapshot, reordered) + } + Err(reason) => { + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + } + } + } else { + (state, args.clone()) + }; + + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { recv, cd, cme, iseq, args: processed_args, state: send_state }); + self.make_equal_to(insn_id, send_direct); + } else if def_type == VM_METHOD_TYPE_IVAR && args.is_empty() { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if self.is_metaclass(klass) && !self.assume_single_ractor_mode(block, state) { + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let id = unsafe { get_cme_def_body_attr_id(cme) }; + + let getivar = self.push_insn(block, Insn::GetIvar { self_val: recv, id, ic: std::ptr::null(), state }); + self.make_equal_to(insn_id, getivar); + } else if let (VM_METHOD_TYPE_ATTRSET, &[val]) = (def_type, args.as_slice()) { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if self.is_metaclass(klass) && !self.assume_single_ractor_mode(block, state) { + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let id = unsafe { get_cme_def_body_attr_id(cme) }; + + self.push_insn(block, Insn::SetIvar { self_val: recv, id, ic: std::ptr::null(), val, state }); + self.make_equal_to(insn_id, val); + } else if def_type == VM_METHOD_TYPE_OPTIMIZED { + let opt_type: OptimizedMethodType = unsafe { get_cme_def_body_optimized_type(cme) }.into(); + match (opt_type, args.as_slice()) { + (OptimizedMethodType::Call, _) => { + if flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KWARG) != 0 { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + let kw_splat = flags & VM_CALL_KW_SPLAT != 0; + let invoke_proc = self.push_insn(block, Insn::InvokeProc { recv, args: args.clone(), state, kw_splat }); + self.make_equal_to(insn_id, invoke_proc); + } + (OptimizedMethodType::StructAref, &[]) | (OptimizedMethodType::StructAset, &[_]) => { + if unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + let index: i32 = unsafe { get_cme_def_body_optimized_index(cme) } + .try_into() + .unwrap(); + // We are going to use an encoding that takes a 4-byte immediate which + // limits the offset to INT32_MAX. + { + let native_index = (index as i64) * (SIZEOF_VALUE as i64); + if native_index > (i32::MAX as i64) { + self.push_insn_id(block, insn_id); continue; + } + } + // Get the profiled type to check if the fields is embedded or heap allocated. + let Some(is_embedded) = self.profiled_type_of_at(recv, frame_state.insn_idx).map(|t| t.flags().is_struct_embedded()) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn_id(block, insn_id); continue; + }; + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + } + // All structs from the same Struct class should have the same + // length. So if our recv is embedded all runtime + // structs of the same class should be as well, and the same is + // true of the converse. + // + // No need for a GuardShape. + if let OptimizedMethodType::StructAset = opt_type { + // We know that all Struct are HeapObject, so no need to insert a GuardType(HeapObject). + recv = self.push_insn(block, Insn::GuardNotFrozen { recv, state }); + } + + let (target, offset) = if is_embedded { + let offset = RUBY_OFFSET_RSTRUCT_AS_ARY + (SIZEOF_VALUE_I32 * index); + (recv, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv, id: ID!(_as_heap), offset: RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * index; + (as_heap, offset) + }; + + let replacement = if let (OptimizedMethodType::StructAset, &[val]) = (opt_type, args.as_slice()) { + self.push_insn(block, Insn::StoreField { recv: target, id: mid, offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv, val }); + val + } else { // StructAref + self.push_insn(block, Insn::LoadField { recv: target, id: mid, offset, return_type: types::BasicObject }) + }; + self.make_equal_to(insn_id, replacement); + }, + _ => { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType::from(opt_type))); + self.push_insn_id(block, insn_id); continue; + }, + }; + } else { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::from(def_type))); + self.push_insn_id(block, insn_id); continue; + } + } + // This doesn't actually optimize Send yet, just replaces the fallback reason to be more precise. + // The actual optimization is done in reduce_send_to_ccall. + Insn::Send { recv, cd, state, .. } => { + let frame_state = self.frame_state(state); + let klass = match self.resolve_receiver_type(recv, self.type_of(recv), frame_state.insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => class, + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => profiled_type.class(), + ReceiverTypeResolution::SkewedMegamorphic { .. } + | ReceiverTypeResolution::Megamorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendMegamorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::Polymorphic => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendPolymorphic); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::NoProfile => { + if get_option!(stats) { + self.set_dynamic_send_reason(insn_id, SendNoProfiles); + } + self.push_insn_id(block, insn_id); + continue; + } + }; + let ci = unsafe { get_call_data_ci(cd) }; // info about the call site + let mid = unsafe { vm_ci_mid(ci) }; + // Do method lookup + let mut cme = unsafe { rb_callable_method_entry(klass, mid) }; + if cme.is_null() { + self.set_dynamic_send_reason(insn_id, SendNotOptimizedMethodType(MethodType::Null)); self.push_insn_id(block, insn_id); continue; } - self.push_insn(block, Insn::PatchPoint(Invariant::MethodRedefined { klass, method: mid })); - let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; - if let Some(expected) = guard_equal_to { - self_val = self.push_insn(block, Insn::GuardBitEquals { val: self_val, expected, state }); + // Load an overloaded cme if applicable. See vm_search_cc(). + // It allows you to use a faster ISEQ if possible. + cme = unsafe { rb_check_overloaded_cme(cme, ci) }; + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; } - let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { self_val, call_info, cd, iseq, args, state }); - self.make_equal_to(insn_id, send_direct); + self.set_dynamic_send_reason(insn_id, SendNotOptimizedMethodType(MethodType::from(def_type))); + self.push_insn_id(block, insn_id); continue; } - Insn::GetConstantPath { ic } => { + Insn::GetConstantPath { ic, state, .. } => { let idlist: *const ID = unsafe { (*ic).segments }; let ice = unsafe { (*ic).entry }; if ice.is_null() { self.push_insn_id(block, insn_id); continue; } let cref_sensitive = !unsafe { (*ice).ic_cref }.is_null(); - let multi_ractor_mode = unsafe { rb_zjit_multi_ractor_p() }; - if cref_sensitive || multi_ractor_mode { + if cref_sensitive || !self.assume_single_ractor_mode(block, state) { self.push_insn_id(block, insn_id); continue; } - // Assume single-ractor mode. - self.push_insn(block, Insn::PatchPoint(Invariant::SingleRactorMode)); // Invalidate output code on any constant writes associated with constants // referenced after the PatchPoint. - self.push_insn(block, Insn::PatchPoint(Invariant::StableConstantNames { idlist })); + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::StableConstantNames { idlist }, state }); let replacement = self.push_insn(block, Insn::Const { val: Const::Value(unsafe { (*ice).value }) }); + self.insn_types[replacement.0] = self.infer_type(replacement); self.make_equal_to(insn_id, replacement); } + Insn::ObjToString { val, cd, state, .. } => { + if self.is_a(val, types::String) { + // behaves differently from `SendWithoutBlock` with `mid:to_s` because ObjToString should not have a patch point for String to_s being redefined + self.make_equal_to(insn_id, val); continue; + } + + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(val, frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); continue + }; + + if recv_type.is_string() { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass: recv_type.class() }, state }); + let guard = self.push_insn(block, Insn::GuardType { val, guard_type: types::String, state }); + // Infer type so AnyToString can fold off this + self.insn_types[guard.0] = self.infer_type(guard); + self.make_equal_to(insn_id, guard); + } else { + let recv = self.push_insn(block, Insn::GuardType { val, guard_type: Type::from_profiled_type(recv_type), state}); + let send_to_s = self.push_insn(block, Insn::SendWithoutBlock { recv, cd, args: vec![], state, reason: ObjToStringNotString }); + self.make_equal_to(insn_id, send_to_s); + } + } + Insn::AnyToString { str, .. } => { + if self.is_a(str, types::String) { + self.make_equal_to(insn_id, str); + } else { + self.push_insn_id(block, insn_id); + } + } + Insn::IsMethodCfunc { val, cd, cfunc, state } if self.type_of(val).ruby_object_known() => { + let class = self.type_of(val).ruby_object().unwrap(); + let cme = unsafe { rb_zjit_vm_search_method(self.iseq.into(), cd as *mut rb_call_data, class) }; + let is_expected_cfunc = unsafe { rb_zjit_cme_is_cfunc(cme, cfunc as *const c_void) }; + let method = unsafe { rb_vm_ci_mid((*cd).ci) }; + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: class, method, cme }, state }); + let replacement = self.push_insn(block, Insn::Const { val: Const::CBool(is_expected_cfunc) }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::ObjectAlloc { val, state } => { + let val_type = self.type_of(val); + if !val_type.is_subtype(types::Class) { + self.push_insn_id(block, insn_id); continue; + } + let Some(class) = val_type.ruby_object() else { + self.push_insn_id(block, insn_id); continue; + }; + // See class_get_alloc_func in object.c; if the class isn't initialized, is + // a singleton class, or has a custom allocator, ObjectAlloc might raise an + // exception or run arbitrary code. + // + // We also need to check if the class is initialized or a singleton before + // trying to read the allocator, otherwise it might raise. + if !unsafe { rb_zjit_class_initialized_p(class) } { + self.push_insn_id(block, insn_id); continue; + } + if unsafe { rb_zjit_singleton_class_p(class) } { + self.push_insn_id(block, insn_id); continue; + } + if !class_has_leaf_allocator(class) { + // Custom, known unsafe, or NULL allocator; could run arbitrary code. + self.push_insn_id(block, insn_id); continue; + } + let replacement = self.push_insn(block, Insn::ObjectAllocClass { class, state }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::NewRange { low, high, flag, state } => { + let low_is_fix = self.is_a(low, types::Fixnum); + let high_is_fix = self.is_a(high, types::Fixnum); + + if low_is_fix || high_is_fix { + let low_fix = self.coerce_to(block, low, types::Fixnum, state); + let high_fix = self.coerce_to(block, high, types::Fixnum, state); + let replacement = self.push_insn(block, Insn::NewRangeFixnum { low: low_fix, high: high_fix, flag, state }); + self.make_equal_to(insn_id, replacement); + self.insn_types[replacement.0] = self.infer_type(replacement); + } else { + self.push_insn_id(block, insn_id); + }; + } + Insn::InvokeSuper { recv, cd, blockiseq, args, state, .. } => { + // Don't handle calls with literal blocks (e.g., super { ... }) + if !blockiseq.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperCallWithBlock); + continue; + } + + let ci = unsafe { get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + assert!(flags & VM_CALL_FCALL != 0); + + // Reject calls with complex argument handling. + let complex_arg_types = VM_CALL_ARGS_SPLAT + | VM_CALL_KW_SPLAT + | VM_CALL_KWARG + | VM_CALL_ARGS_BLOCKARG + | VM_CALL_FORWARDING; + + if (flags & complex_arg_types) != 0 { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperComplexArgsPass); + continue; + } + + let frame_state = self.frame_state(state); + + // Get the profiled CME from the current method. + let Some(profiles) = self.profiles.as_ref() else { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNoProfiles); + continue; + }; + + let Some(current_cme) = profiles.payload.profile.get_super_method_entry(frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); + + // The absence of the super CME could be due to a missing profile, but + // if we've made it this far the value would have been deleted, indicating + // that the call is at least polymorphic and possibly megamorphic. + self.set_dynamic_send_reason(insn_id, SuperPolymorphic); + continue; + }; + + // Get defined_class and method ID from the profiled CME. + let current_defined_class = unsafe { (*current_cme).defined_class }; + let mid = unsafe { get_def_original_id((*current_cme).def) }; + + // Compute superclass: RCLASS_SUPER(RCLASS_ORIGIN(defined_class)) + let superclass = unsafe { rb_class_get_superclass(RCLASS_ORIGIN(current_defined_class)) }; + if superclass.nil_p() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperClassNotFound); + continue; + } + + // Look up the super method. + let super_cme = unsafe { rb_callable_method_entry(superclass, mid) }; + if super_cme.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetNotFound); + continue; + } + + // Check if it's an ISEQ method; bail if it isn't. + let def_type = unsafe { get_cme_def_type(super_cme) }; + if def_type != VM_METHOD_TYPE_ISEQ { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNotOptimizedMethodType(MethodType::from(def_type))); + continue; + } + + // Check if the super method's parameters support direct send. + // If not, we can't do direct dispatch. + let super_iseq = unsafe { get_def_iseq_ptr((*super_cme).def) }; + if !can_direct_send(self, block, super_iseq, insn_id, args.as_slice()) { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetComplexArgsPass); + continue; + } + + // Add PatchPoint for method redefinition. + self.push_insn(block, Insn::PatchPoint { + invariant: Invariant::MethodRedefined { + klass: unsafe { (*super_cme).defined_class }, + method: mid, + cme: super_cme + }, + state + }); + + // Guard that we're calling `super` from the expected method context. + self.push_insn(block, Insn::GuardSuperMethodEntry { cme: current_cme, state }); + + // Guard that no block is being passed (implicit or explicit). + let block_handler = self.push_insn(block, Insn::GetBlockHandler); + self.push_insn(block, Insn::GuardBitEquals { + val: block_handler, + expected: Const::Value(VALUE(VM_BLOCK_HANDLER_NONE as usize)), + reason: SideExitReason::UnhandledBlockArg, + state + }); + + // Use SendWithoutBlockDirect with the super method's CME and ISEQ. + let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { + recv, + cd, + cme: super_cme, + iseq: super_iseq, + args, + state + }); + self.make_equal_to(insn_id, send_direct); + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + self.infer_types(); + } + + fn inline(&mut self) { + for block in self.rpo() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + // Reject block ISEQs to avoid autosplat and other block parameter complications. + Insn::SendWithoutBlockDirect { recv, iseq, cd, args, state, .. } => { + let call_info = unsafe { (*cd).ci }; + let ci_flags = unsafe { vm_ci_flag(call_info) }; + // .send call is not currently supported for builtins + if ci_flags & VM_CALL_OPT_SEND != 0 { + self.push_insn_id(block, insn_id); continue; + } + let Some(value) = iseq_get_return_value(iseq, None, ci_flags) else { + self.push_insn_id(block, insn_id); continue; + }; + match value { + IseqReturn::LocalVariable(idx) => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + self.make_equal_to(insn_id, args[idx as usize]); + } + IseqReturn::Value(value) => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + let replacement = self.push_insn(block, Insn::Const { val: Const::Value(value) }); + self.make_equal_to(insn_id, replacement); + } + IseqReturn::Receiver => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + self.make_equal_to(insn_id, recv); + } + IseqReturn::InvokeLeafBuiltin(bf, return_type) => { + self.push_insn(block, Insn::IncrCounter(Counter::inline_iseq_optimized_send_count)); + let replacement = self.push_insn(block, Insn::InvokeBuiltin { + bf, + recv, + args: vec![recv], + state, + leaf: true, + return_type, + }); + self.make_equal_to(insn_id, replacement); + } + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + self.infer_types(); + } + + fn load_shape(&mut self, block: BlockId, recv: InsnId) -> InsnId { + self.push_insn(block, Insn::LoadField { + recv, + id: ID!(_shape_id), + offset: unsafe { rb_shape_id_offset() } as i32, + return_type: types::CShape + }) + } + + fn guard_shape(&mut self, block: BlockId, val: InsnId, expected: ShapeId, state: InsnId) -> InsnId { + self.push_insn(block, Insn::GuardBitEquals { + val, + expected: Const::CShape(expected), + reason: SideExitReason::GuardShape(expected), + state + }) + } + + fn optimize_getivar(&mut self) { + for block in self.rpo() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::GetIvar { self_val, id, ic: _, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn(block, Insn::IncrCounter(Counter::getivar_fallback_not_monomorphic)); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.push_insn(block, Insn::IncrCounter(Counter::getivar_fallback_immediate)); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if recv_type.shape().is_too_complex() { + // too-complex shapes can't use index access + self.push_insn(block, Insn::IncrCounter(Counter::getivar_fallback_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: types::HeapBasicObject, state }); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state); + let mut ivar_index: u16 = 0; + let replacement = if ! unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }) + } else if !recv_type.flags().is_t_object() { + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + let ivar_index_insn: InsnId = self.push_insn(block, Insn::Const { val: Const::CUInt16(ivar_index as u16) }); + self.push_insn(block, Insn::CCall { + cfunc: rb_ivar_get_at_no_ractor_check as *const u8, + recv: self_val, + args: vec![ivar_index_insn], + name: ID!(rb_ivar_get_at_no_ractor_check), + return_type: types::BasicObject, + elidable: true }) + } else if recv_type.flags().is_embedded() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + self.push_insn(block, Insn::LoadField { recv: self_val, id, offset, return_type: types::BasicObject }) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: ID!(_as_heap), offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, return_type: types::CPtr }); + + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + self.push_insn(block, Insn::LoadField { recv: as_heap, id, offset, return_type: types::BasicObject }) + }; + self.make_equal_to(insn_id, replacement); + } + Insn::DefinedIvar { self_val, id, pushval, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_not_monomorphic)); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_immediate)); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_not_t_object)); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_too_complex() { + // too-complex shapes can't use index access + self.push_insn(block, Insn::IncrCounter(Counter::definedivar_fallback_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: types::HeapBasicObject, state }); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state); + let mut ivar_index: u16 = 0; + let replacement = if unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + self.push_insn(block, Insn::Const { val: Const::Value(pushval) }) + } else { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }) + }; + self.make_equal_to(insn_id, replacement); + } + Insn::SetIvar { self_val, id, val, state, ic: _ } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_not_monomorphic)); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_immediate)); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_not_t_object)); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_too_complex() { + // too-complex shapes can't use index access + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_frozen() { + // Can't set ivars on frozen objects + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_frozen)); + self.push_insn_id(block, insn_id); continue; + } + let mut ivar_index: u16 = 0; + let mut next_shape_id = recv_type.shape(); + if !unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // Current shape does not contain this ivar; do a shape transition. + let current_shape_id = recv_type.shape(); + let class = recv_type.class(); + // We're only looking at T_OBJECT so ignore all of the imemo stuff. + assert!(recv_type.flags().is_t_object()); + next_shape_id = ShapeId(unsafe { rb_shape_transition_add_ivar_no_warnings(class, current_shape_id.0, id) }); + // If the VM ran out of shapes, or this class generated too many leaf, + // it may be de-optimized into OBJ_TOO_COMPLEX_SHAPE (hash-table). + let new_shape_too_complex = unsafe { rb_jit_shape_too_complex_p(next_shape_id.0) }; + // TODO(max): Is it OK to bail out here after making a shape transition? + if new_shape_too_complex { + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_new_shape_too_complex)); + self.push_insn_id(block, insn_id); continue; + } + let ivar_result = unsafe { rb_shape_get_iv_index(next_shape_id.0, id, &mut ivar_index) }; + assert!(ivar_result, "New shape must have the ivar index"); + let current_capacity = unsafe { rb_jit_shape_capacity(current_shape_id.0) }; + let next_capacity = unsafe { rb_jit_shape_capacity(next_shape_id.0) }; + // If the new shape has a different capacity, or is TOO_COMPLEX, we'll have to + // reallocate it. + let needs_extension = next_capacity != current_capacity; + if needs_extension { + self.push_insn(block, Insn::IncrCounter(Counter::setivar_fallback_new_shape_needs_extension)); + self.push_insn_id(block, insn_id); continue; + } + // Fall through to emitting the ivar write + } + let self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: types::HeapBasicObject, state }); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state); + // Current shape contains this ivar + let (ivar_storage, offset) = if recv_type.flags().is_embedded() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + (self_val, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: ID!(_as_heap), offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + (as_heap, offset) + }; + self.push_insn(block, Insn::StoreField { recv: ivar_storage, id, offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv: self_val, val }); + if next_shape_id != recv_type.shape() { + // Write the new shape ID + let shape_id = self.push_insn(block, Insn::Const { val: Const::CShape(next_shape_id) }); + let shape_id_offset = unsafe { rb_shape_id_offset() }; + self.push_insn(block, Insn::StoreField { recv: self_val, id: ID!(_shape_id), offset: shape_id_offset, val: shape_id }); + } + } _ => { self.push_insn_id(block, insn_id); } } } @@ -1147,18 +3628,27 @@ impl Function { self.infer_types(); } + fn gen_patch_points_for_optimized_ccall(&mut self, block: BlockId, recv_class: VALUE, method_id: ID, cme: *const rb_callable_method_entry_struct, state: InsnId) { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state }); + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: recv_class, method: method_id, cme }, state }); + } + /// Optimize SendWithoutBlock that land in a C method to a direct CCall without /// runtime lookup. fn optimize_c_calls(&mut self) { - // Try to reduce one SendWithoutBlock to a CCall - fn reduce_to_ccall( + if unsafe { rb_zjit_method_tracing_currently_enabled() } { + return; + } + + // Try to reduce a Send insn to a CCallWithFrame + fn reduce_send_to_ccall( fun: &mut Function, block: BlockId, self_type: Type, send: Insn, send_insn_id: InsnId, ) -> Result<(), ()> { - let Insn::SendWithoutBlock { mut self_val, cd, mut args, state, .. } = send else { + let Insn::Send { mut recv, cd, blockiseq, args, state, .. } = send else { return Err(()); }; @@ -1167,34 +3657,56 @@ impl Function { let method_id = unsafe { rb_vm_ci_mid(call_info) }; // If we have info about the class of the receiver - // - // TODO(alan): there was a seemingly a miscomp here if you swap with - // `inexact_ruby_class`. Theoretically it can call a method too general - // for the receiver. Confirm and add a test. - let (recv_class, guard_type) = if let Some(klass) = self_type.runtime_exact_ruby_class() { - (klass, None) - } else { - let iseq_insn_idx = fun.frame_state(state).insn_idx; - let Some(recv_type) = fun.profiled_type_of_at(self_val, iseq_insn_idx) else { return Err(()) }; - let Some(recv_class) = recv_type.exact_ruby_class() else { return Err(()) }; - (recv_class, Some(recv_type.unspecialized())) + let iseq_insn_idx = fun.frame_state(state).insn_idx; + let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type} => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()), }; // Do method lookup - let method = unsafe { rb_callable_method_entry(recv_class, method_id) }; - if method.is_null() { + let cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; + if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedMethodType(MethodType::Null)); return Err(()); } // Filter for C methods - let def_type = unsafe { get_cme_def_type(method) }; + // TODO(max): Handle VM_METHOD_TYPE_ALIAS + let def_type = unsafe { get_cme_def_type(cme) }; if def_type != VM_METHOD_TYPE_CFUNC { return Err(()); } + + let ci_flags = unsafe { vm_ci_flag(call_info) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, ci_flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedNeedPermission); + return Err(()); + } + } + + // When seeing &block argument, fall back to dynamic dispatch for now + // TODO: Support block forwarding + if unspecializable_c_call_type(ci_flags) { + fun.count_complex_call_features(block, ci_flags); + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } + + let blockiseq = if blockiseq.is_null() { None } else { Some(blockiseq) }; + + let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; // Find the `argc` (arity) of the C method, which describes the parameters it expects - let cfunc = unsafe { get_cme_def_body_cfunc(method) }; let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast(); + match cfunc_argc { 0.. => { // (self, arg0, arg1, ..., argc) form @@ -1204,38 +3716,307 @@ impl Function { return Err(()); } - // Filter for a leaf and GC free function - use crate::cruby_methods::FnProperties; - let Some(FnProperties { leaf: true, no_gc: true, return_type, elidable }) = - ZJITState::get_method_annotations().get_cfunc_properties(method) - else { + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); return Err(()); - }; + } + + // Commit to the replacement. Put PatchPoint. + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Emit a call + let cfunc = unsafe { get_mct_func(cfunc) }.cast(); + + let name = rust_str_to_id(&qualified_method_name(unsafe { (*cme).owner }, unsafe { (*cme).called_id })); + let ccall = fun.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc, + recv, + args, + cme, + name, + state, + return_type: types::BasicObject, + elidable: false, + blockiseq, + }); + fun.make_equal_to(send_insn_id, ccall); + Ok(()) + } + // Variadic method + -1 => { + // The method gets a pointer to the first argument + // func(int argc, VALUE *argv, VALUE recv) + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + if get_option!(stats) { + count_not_inlined_cfunc(fun, block, cme); + } + + let ccall = fun.push_insn(block, Insn::CCallVariadic { + cfunc: cfunc_ptr, + recv, + args, + cme, + name: method_id, + state, + return_type: types::BasicObject, + elidable: false, + blockiseq + }); + + fun.make_equal_to(send_insn_id, ccall); + Ok(()) + } + -2 => { + // (self, args_ruby_array) + fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic); + Err(()) + } + _ => unreachable!("unknown cfunc kind: argc={argc}") + } + } + + // Try to reduce a SendWithoutBlock insn to a CCall/CCallWithFrame + fn reduce_send_without_block_to_ccall( + fun: &mut Function, + block: BlockId, + self_type: Type, + send: Insn, + send_insn_id: InsnId, + ) -> Result<(), ()> { + let Insn::SendWithoutBlock { mut recv, cd, args, state, .. } = send else { + return Err(()); + }; + + let call_info = unsafe { (*cd).ci }; + let argc = unsafe { vm_ci_argc(call_info) }; + let method_id = unsafe { rb_vm_ci_mid(call_info) }; + + // If we have info about the class of the receiver + let iseq_insn_idx = fun.frame_state(state).insn_idx; + let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()), + }; + + // Do method lookup + let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; + if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::Null)); + return Err(()); + } + + // Filter for C methods + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + if def_type != VM_METHOD_TYPE_CFUNC { + return Err(()); + } + + let ci_flags = unsafe { vm_ci_flag(call_info) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, ci_flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedNeedPermission); + return Err(()); + } + } + + // Find the `argc` (arity) of the C method, which describes the parameters it expects + let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; + let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + match cfunc_argc { + 0.. => { + // (self, arg0, arg1, ..., argc) form + // + // Bail on argc mismatch + if argc != cfunc_argc as u32 { + return Err(()); + } - let ci_flags = unsafe { vm_ci_flag(call_info) }; // Filter for simple call sites (i.e. no splats etc.) - if ci_flags & VM_CALL_ARGS_SIMPLE != 0 { - // Commit to the replacement. Put PatchPoint. - fun.push_insn(block, Insn::PatchPoint(Invariant::MethodRedefined { klass: recv_class, method: method_id })); - if let Some(guard_type) = guard_type { - // Guard receiver class - self_val = fun.push_insn(block, Insn::GuardType { val: self_val, guard_type, state }); + if ci_flags & VM_CALL_ARGS_SIMPLE == 0 { + fun.count_complex_call_features(block, ci_flags); + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + // Commit to the replacement. Put PatchPoint. + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); + if props.is_none() && get_option!(stats) { + count_not_annotated_cfunc(fun, block, cme); + } + let props = props.unwrap_or_default(); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Try inlining the cfunc into HIR + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); } - let cfun = unsafe { get_mct_func(cfunc) }.cast(); - let mut cfunc_args = vec![self_val]; - cfunc_args.append(&mut args); - let ccall = fun.push_insn(block, Insn::CCall { cfun, args: cfunc_args, name: method_id, return_type, elidable }); - fun.make_equal_to(send_insn_id, ccall); + fun.remove_block(tmp_block); return Ok(()); } + + // No inlining; emit a call + let cfunc = unsafe { get_mct_func(cfunc) }.cast(); + let name = rust_str_to_id(&qualified_method_name(unsafe { (*cme).owner }, unsafe { (*cme).called_id })); + let return_type = props.return_type; + let elidable = props.elidable; + // Filter for a leaf and GC free function + if props.leaf && props.no_gc { + fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + let ccall = fun.push_insn(block, Insn::CCall { cfunc, recv, args, name, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + } else { + if get_option!(stats) { + count_not_inlined_cfunc(fun, block, cme); + } + let ccall = fun.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc, + recv, + args, + cme, + name, + state, + return_type, + elidable, + blockiseq: None, + }); + fun.make_equal_to(send_insn_id, ccall); + } + + return Ok(()); } + // Variadic method -1 => { - // (argc, argv, self) parameter form - // Falling through for now + // The method gets a pointer to the first argument + // func(int argc, VALUE *argv, VALUE recv) + let ci_flags = unsafe { vm_ci_flag(call_info) }; + if ci_flags & VM_CALL_ARGS_SIMPLE == 0 { + // TODO(alan): Add fun.count_complex_call_features() here without double + // counting splat + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } else { + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + let cfunc = unsafe { get_mct_func(cfunc) }.cast(); + let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); + if props.is_none() && get_option!(stats) { + count_not_annotated_cfunc(fun, block, cme); + } + let props = props.unwrap_or_default(); + + // Try inlining the cfunc into HIR + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); + } + + // No inlining; emit a call + if get_option!(stats) { + count_not_inlined_cfunc(fun, block, cme); + } + let return_type = props.return_type; + let elidable = props.elidable; + let name = rust_str_to_id(&qualified_method_name(unsafe { (*cme).owner }, unsafe { (*cme).called_id })); + let ccall = fun.push_insn(block, Insn::CCallVariadic { + cfunc, + recv, + args, + cme, + name, + state, + return_type, + elidable, + blockiseq: None, + }); + + fun.make_equal_to(send_insn_id, ccall); + return Ok(()) + } + + // Fall through for complex cases (splat, kwargs, etc.) } -2 => { // (self, args_ruby_array) parameter form // Falling through for now + fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockCfuncArrayVariadic); } _ => unreachable!("unknown cfunc kind: argc={argc}") } @@ -1243,15 +4024,78 @@ impl Function { Err(()) } + fn qualified_method_name(class: VALUE, method_id: ID) -> String { + let method_name = method_id.contents_lossy(); + // rb_zjit_singleton_class_p also checks if it's a class + if unsafe { rb_zjit_singleton_class_p(class) } { + let class_name = get_class_name(unsafe { rb_class_attached_object(class) }); + format!("{class_name}.{method_name}") + } else { + let class_name = get_class_name(class); + format!("{class_name}#{method_name}") + } + } + + fn count_not_inlined_cfunc(fun: &mut Function, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_inlined_cfunc_counter_pointers = ZJITState::get_not_inlined_cfunc_counter_pointers(); + let counter_ptr = not_inlined_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + fun.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + fn count_not_annotated_cfunc(fun: &mut Function, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_annotated_cfunc_counter_pointers = ZJITState::get_not_annotated_cfunc_counter_pointers(); + let counter_ptr = not_annotated_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + fun.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + for block in self.rpo() { let old_insns = std::mem::take(&mut self.blocks[block.0].insns); assert!(self.blocks[block.0].insns.is_empty()); for insn_id in old_insns { - if let send @ Insn::SendWithoutBlock { self_val, .. } = self.find(insn_id) { - let self_type = self.type_of(self_val); - if reduce_to_ccall(self, block, self_type, send, insn_id).is_ok() { - continue; + let send = self.find(insn_id); + match send { + send @ Insn::SendWithoutBlock { recv, .. } => { + let recv_type = self.type_of(recv); + if reduce_send_without_block_to_ccall(self, block, recv_type, send, insn_id).is_ok() { + continue; + } + } + send @ Insn::Send { recv, .. } => { + let recv_type = self.type_of(recv); + if reduce_send_to_ccall(self, block, recv_type, send, insn_id).is_ok() { + continue; + } } + Insn::InvokeBuiltin { bf, recv, args, state, .. } => { + let props = ZJITState::get_method_annotations().get_builtin_properties(&bf).unwrap_or_default(); + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.push_insn(block, Insn::IncrCounter(Counter::inline_cfunc_optimized_send_count)); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } + } + _ => {} } self.push_insn_id(block, insn_id); } @@ -1259,6 +4103,22 @@ impl Function { self.infer_types(); } + /// Fold a binary operator on fixnums. + fn fold_fixnum_bop(&mut self, insn_id: InsnId, left: InsnId, right: InsnId, f: impl FnOnce(Option<i64>, Option<i64>) -> Option<i64>) -> InsnId { + f(self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) + .filter(|&n| n >= (RUBY_FIXNUM_MIN as i64) && n <= RUBY_FIXNUM_MAX as i64) + .map(|n| self.new_insn(Insn::Const { val: Const::Value(VALUE::fixnum_from_isize(n as isize)) })) + .unwrap_or(insn_id) + } + + /// Fold a binary predicate on fixnums. + fn fold_fixnum_pred(&mut self, insn_id: InsnId, left: InsnId, right: InsnId, f: impl FnOnce(Option<i64>, Option<i64>) -> Option<bool>) -> InsnId { + f(self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) + .map(|b| if b { Qtrue } else { Qfalse }) + .map(|b| self.new_insn(Insn::Const { val: Const::Value(b) })) + .unwrap_or(insn_id) + } + /// Use type information left by `infer_types` to fold away operations that can be evaluated at compile-time. /// /// It can fold fixnum math, truthiness tests, and branches with constant conditionals. @@ -1279,40 +4139,108 @@ impl Function { // Don't bother re-inferring the type of val; we already know it. continue; } - Insn::FixnumAdd { left, right, .. } => { - match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { - (Some(l), Some(r)) => { - let result = l + r; - if result >= (RUBY_FIXNUM_MIN as i64) && result <= (RUBY_FIXNUM_MAX as i64) { - self.new_insn(Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(result as usize)) }) - } else { - // Instead of allocating a Bignum at compile-time, defer the add and allocation to run-time. - insn_id - } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::BasicObject) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const VALUE; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::Value(val) }) } _ => insn_id, } } - Insn::FixnumLt { left, right, .. } => { - match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { - (Some(l), Some(r)) => { - if l < r { - self.new_insn(Insn::Const { val: Const::Value(Qtrue) }) - } else { - self.new_insn(Insn::Const { val: Const::Value(Qfalse) }) - } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::CShape) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const u32; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::CShape(ShapeId(val)) }) } _ => insn_id, } } + Insn::GuardBitEquals { val, expected, .. } => { + let recv_type = self.type_of(val); + if recv_type.has_value(expected) { + continue; + } else { + insn_id + } + } + Insn::AnyToString { str, .. } if self.is_a(str, types::String) => { + self.make_equal_to(insn_id, str); + // Don't bother re-inferring the type of str; we already know it. + continue; + } + Insn::FixnumAdd { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_add(r), + _ => None, + }) + } + Insn::FixnumSub { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_sub(r), + _ => None, + }) + } + Insn::FixnumMult { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => l.checked_mul(r), + (Some(0), _) | (_, Some(0)) => Some(0), + _ => None, + }) + } Insn::FixnumEq { left, right, .. } => { - match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { - (Some(l), Some(r)) => { - if l == r { - self.new_insn(Insn::Const { val: Const::Value(Qtrue) }) - } else { - self.new_insn(Insn::Const { val: Const::Value(Qfalse) }) - } + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l == r), + _ => None, + }) + } + Insn::FixnumNeq { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l != r), + _ => None, + }) + } + Insn::FixnumLt { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l < r), + _ => None, + }) + } + Insn::FixnumLe { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l <= r), + _ => None, + }) + } + Insn::FixnumGt { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l > r), + _ => None, + }) + } + Insn::FixnumGe { left, right, .. } => { + self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l >= r), + _ => None, + }) + } + Insn::ArrayAref { array, index } + if self.type_of(array).ruby_object_known() + && self.type_of(index).is_subtype(types::CInt64) => { + let array_obj = self.type_of(array).ruby_object().unwrap(); + match (array_obj.is_frozen(), self.type_of(index).cint64_value()) { + (true, Some(index)) => { + let val = unsafe { rb_yarv_ary_entry_internal(array_obj, index) }; + self.new_insn(Insn::Const { val: Const::Value(val) }) } _ => insn_id, } @@ -1337,11 +4265,9 @@ impl Function { }; // If we're adding a new instruction, mark the two equivalent in the union-find and // do an incremental flow typing of the new instruction. - if insn_id != replacement_id { + if insn_id != replacement_id && self.insns[replacement_id.0].has_output() { self.make_equal_to(insn_id, replacement_id); - if self.insns[replacement_id.0].has_output() { - self.insn_types[replacement_id.0] = self.infer_type(replacement_id); - } + self.insn_types[replacement_id.0] = self.infer_type(replacement_id); } new_insns.push(replacement_id); // If we've just folded an IfTrue into a Jump, for example, don't bother copying @@ -1354,6 +4280,263 @@ impl Function { } } + fn worklist_traverse_single_insn(&self, insn: &Insn, worklist: &mut VecDeque<InsnId>) { + match insn { + &Insn::Const { .. } + | &Insn::Param + | &Insn::EntryPoint { .. } + | &Insn::LoadPC + | &Insn::LoadEC + | &Insn::LoadSelf + | &Insn::GetLocal { .. } + | &Insn::GetBlockHandler + | &Insn::PutSpecialObject { .. } + | &Insn::IsBlockGiven + | &Insn::IncrCounter(_) + | &Insn::IncrCounterPtr { .. } => + {} + &Insn::PatchPoint { state, .. } + | &Insn::CheckInterrupts { state } + | &Insn::GetConstantPath { ic: _, state } => { + worklist.push_back(state); + } + &Insn::FixnumBitCheck { val, index: _ } => { + worklist.push_back(val) + } + &Insn::ArrayMax { ref elements, state } + | &Insn::ArrayHash { ref elements, state } + | &Insn::NewHash { ref elements, state } + | &Insn::NewArray { ref elements, state } => { + worklist.extend(elements); + worklist.push_back(state); + } + &Insn::ArrayInclude { ref elements, target, state } => { + worklist.extend(elements); + worklist.push_back(target); + worklist.push_back(state); + } + &Insn::ArrayPackBuffer { ref elements, fmt, buffer, state } => { + worklist.extend(elements); + worklist.push_back(fmt); + worklist.push_back(buffer); + worklist.push_back(state); + } + &Insn::DupArrayInclude { target, state, .. } => { + worklist.push_back(target); + worklist.push_back(state); + } + &Insn::NewRange { low, high, state, .. } + | &Insn::NewRangeFixnum { low, high, state, .. } => { + worklist.push_back(low); + worklist.push_back(high); + worklist.push_back(state); + } + &Insn::StringConcat { ref strings, state, .. } => { + worklist.extend(strings); + worklist.push_back(state); + } + &Insn::StringGetbyte { string, index } => { + worklist.push_back(string); + worklist.push_back(index); + } + &Insn::StringSetbyteFixnum { string, index, value } => { + worklist.push_back(string); + worklist.push_back(index); + worklist.push_back(value); + } + &Insn::StringAppend { recv, other, state } + | &Insn::StringAppendCodepoint { recv, other, state } + => { + worklist.push_back(recv); + worklist.push_back(other); + worklist.push_back(state); + } + &Insn::ToRegexp { ref values, state, .. } => { + worklist.extend(values); + worklist.push_back(state); + } + | &Insn::Return { val } + | &Insn::Test { val } + | &Insn::SetLocal { val, .. } + | &Insn::BoxBool { val } + | &Insn::IsNil { val } => + worklist.push_back(val), + &Insn::SetGlobal { val, state, .. } + | &Insn::Defined { v: val, state, .. } + | &Insn::StringIntern { val, state } + | &Insn::StringCopy { val, state, .. } + | &Insn::ObjectAlloc { val, state } + | &Insn::GuardType { val, state, .. } + | &Insn::GuardTypeNot { val, state, .. } + | &Insn::GuardBitEquals { val, state, .. } + | &Insn::GuardShape { val, state, .. } + | &Insn::GuardNotFrozen { recv: val, state } + | &Insn::GuardNotShared { recv: val, state } + | &Insn::ToArray { val, state } + | &Insn::IsMethodCfunc { val, state, .. } + | &Insn::ToNewArray { val, state } + | &Insn::BoxFixnum { val, state } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::GuardGreaterEq { left, right, state } => { + worklist.push_back(left); + worklist.push_back(right); + worklist.push_back(state); + } + &Insn::GuardLess { left, right, state } => { + worklist.push_back(left); + worklist.push_back(right); + worklist.push_back(state); + } + Insn::Snapshot { state } => { + worklist.extend(&state.stack); + worklist.extend(&state.locals); + } + &Insn::FixnumAdd { left, right, state } + | &Insn::FixnumSub { left, right, state } + | &Insn::FixnumMult { left, right, state } + | &Insn::FixnumDiv { left, right, state } + | &Insn::FixnumMod { left, right, state } + | &Insn::ArrayExtend { left, right, state } + | &Insn::FixnumLShift { left, right, state } + => { + worklist.push_back(left); + worklist.push_back(right); + worklist.push_back(state); + } + &Insn::FixnumLt { left, right } + | &Insn::FixnumLe { left, right } + | &Insn::FixnumGt { left, right } + | &Insn::FixnumGe { left, right } + | &Insn::FixnumEq { left, right } + | &Insn::FixnumNeq { left, right } + | &Insn::FixnumAnd { left, right } + | &Insn::FixnumOr { left, right } + | &Insn::FixnumXor { left, right } + | &Insn::FixnumRShift { left, right } + | &Insn::IsBitEqual { left, right } + | &Insn::IsBitNotEqual { left, right } + => { + worklist.push_back(left); + worklist.push_back(right); + } + &Insn::Jump(BranchEdge { ref args, .. }) => worklist.extend(args), + &Insn::IfTrue { val, target: BranchEdge { ref args, .. } } | &Insn::IfFalse { val, target: BranchEdge { ref args, .. } } => { + worklist.push_back(val); + worklist.extend(args); + } + &Insn::ArrayDup { val, state } + | &Insn::Throw { val, state, .. } + | &Insn::HashDup { val, state } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::ArrayAref { array, index } => { + worklist.push_back(array); + worklist.push_back(index); + } + &Insn::ArrayAset { array, index, val } => { + worklist.push_back(array); + worklist.push_back(index); + worklist.push_back(val); + } + &Insn::ArrayPop { array, state } => { + worklist.push_back(array); + worklist.push_back(state); + } + &Insn::ArrayLength { array } => { + worklist.push_back(array); + } + &Insn::HashAref { hash, key, state } => { + worklist.push_back(hash); + worklist.push_back(key); + worklist.push_back(state); + } + &Insn::HashAset { hash, key, val, state } => { + worklist.push_back(hash); + worklist.push_back(key); + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::Send { recv, ref args, state, .. } + | &Insn::SendForward { recv, ref args, state, .. } + | &Insn::SendWithoutBlock { recv, ref args, state, .. } + | &Insn::CCallVariadic { recv, ref args, state, .. } + | &Insn::CCallWithFrame { recv, ref args, state, .. } + | &Insn::SendWithoutBlockDirect { recv, ref args, state, .. } + | &Insn::InvokeBuiltin { recv, ref args, state, .. } + | &Insn::InvokeSuper { recv, ref args, state, .. } + | &Insn::InvokeProc { recv, ref args, state, .. } => { + worklist.push_back(recv); + worklist.extend(args); + worklist.push_back(state); + } + &Insn::InvokeBlock { ref args, state, .. } => { + worklist.extend(args); + worklist.push_back(state) + } + &Insn::CCall { recv, ref args, .. } => { + worklist.push_back(recv); + worklist.extend(args); + } + &Insn::GetIvar { self_val, state, .. } | &Insn::DefinedIvar { self_val, state, .. } => { + worklist.push_back(self_val); + worklist.push_back(state); + } + &Insn::SetIvar { self_val, val, state, .. } => { + worklist.push_back(self_val); + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::GetClassVar { state, .. } => { + worklist.push_back(state); + } + &Insn::SetClassVar { val, state, .. } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::ArrayPush { array, val, state } => { + worklist.push_back(array); + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::ObjToString { val, state, .. } => { + worklist.push_back(val); + worklist.push_back(state); + } + &Insn::AnyToString { val, str, state, .. } => { + worklist.push_back(val); + worklist.push_back(str); + worklist.push_back(state); + } + &Insn::LoadField { recv, .. } => { + worklist.push_back(recv); + } + &Insn::StoreField { recv, val, .. } + | &Insn::WriteBarrier { recv, val } => { + worklist.push_back(recv); + worklist.push_back(val); + } + &Insn::GuardBlockParamProxy { state, .. } | + &Insn::GuardSuperMethodEntry { state, .. } | + &Insn::GetGlobal { state, .. } | + &Insn::GetSpecialSymbol { state, .. } | + &Insn::GetSpecialNumber { state, .. } | + &Insn::ObjectAllocClass { state, .. } | + &Insn::SideExit { state, .. } => worklist.push_back(state), + &Insn::UnboxFixnum { val } => worklist.push_back(val), + &Insn::FixnumAref { recv, index } => { + worklist.push_back(recv); + worklist.push_back(index); + } + &Insn::IsA { val, class } => { + worklist.push_back(val); + worklist.push_back(class); + } + } + } + /// Remove instructions that do not have side effects and are not referenced by any other /// instruction. fn eliminate_dead_code(&mut self) { @@ -1363,106 +4546,108 @@ impl Function { // otherwise necessary to keep around for block_id in &rpo { for insn_id in &self.blocks[block_id.0].insns { - let insn = &self.insns[insn_id.0]; - if insn.has_effects() { + if !&self.insns[insn_id.0].is_elidable() { worklist.push_back(*insn_id); } } } - let mut necessary = vec![false; self.insns.len()]; + let mut necessary = InsnSet::with_capacity(self.insns.len()); // Now recursively traverse their data dependencies and mark those as necessary while let Some(insn_id) = worklist.pop_front() { - if necessary[insn_id.0] { continue; } - necessary[insn_id.0] = true; - match self.find(insn_id) { - Insn::PutSelf | Insn::Const { .. } | Insn::Param { .. } - | Insn::PatchPoint(..) | Insn::GetConstantPath { .. } => - {} - Insn::ArrayMax { elements, state } - | Insn::NewArray { elements, state } => { - worklist.extend(elements); - worklist.push_back(state); - } - Insn::StringCopy { val } - | Insn::StringIntern { val } - | Insn::Return { val } - | Insn::Defined { v: val, .. } - | Insn::Test { val } => - worklist.push_back(val), - Insn::GuardType { val, state, .. } - | Insn::GuardBitEquals { val, state, .. } => { - worklist.push_back(val); - worklist.push_back(state); - } - Insn::ArraySet { array, val, .. } => { - worklist.push_back(array); - worklist.push_back(val); - } - Insn::Snapshot { state } => { - worklist.extend(&state.stack); - worklist.extend(&state.locals); - } - Insn::FixnumAdd { left, right, state } - | Insn::FixnumSub { left, right, state } - | Insn::FixnumMult { left, right, state } - | Insn::FixnumDiv { left, right, state } - | Insn::FixnumMod { left, right, state } - => { - worklist.push_back(left); - worklist.push_back(right); - worklist.push_back(state); - } - Insn::FixnumLt { left, right } - | Insn::FixnumLe { left, right } - | Insn::FixnumGt { left, right } - | Insn::FixnumGe { left, right } - | Insn::FixnumEq { left, right } - | Insn::FixnumNeq { left, right } - => { - worklist.push_back(left); - worklist.push_back(right); - } - Insn::Jump(BranchEdge { args, .. }) => worklist.extend(args), - Insn::IfTrue { val, target: BranchEdge { args, .. } } | Insn::IfFalse { val, target: BranchEdge { args, .. } } => { - worklist.push_back(val); - worklist.extend(args); - } - Insn::ArrayDup { val , state } => { - worklist.push_back(val); - worklist.push_back(state); - } - Insn::Send { self_val, args, state, .. } - | Insn::SendWithoutBlock { self_val, args, state, .. } - | Insn::SendWithoutBlockDirect { self_val, args, state, .. } => { - worklist.push_back(self_val); - worklist.extend(args); - worklist.push_back(state); - } - Insn::CCall { args, .. } => worklist.extend(args), - } + if necessary.get(insn_id) { continue; } + necessary.insert(insn_id); + self.worklist_traverse_single_insn(&self.find(insn_id), &mut worklist); } // Now remove all unnecessary instructions for block_id in &rpo { - self.blocks[block_id.0].insns.retain(|insn_id| necessary[insn_id.0]); + self.blocks[block_id.0].insns.retain(|&insn_id| necessary.get(insn_id)); } } + fn absorb_dst_block(&mut self, num_in_edges: &[u32], block: BlockId) -> bool { + let Some(terminator_id) = self.blocks[block.0].insns.last() + else { return false }; + let Insn::Jump(BranchEdge { target, args }) = self.find(*terminator_id) + else { return false }; + if target == block { + // Can't absorb self + return false; + } + if num_in_edges[target.0] != 1 { + // Can't absorb block if it's the target of more than one branch + return false; + } + // Link up params with block args + let params = std::mem::take(&mut self.blocks[target.0].params); + assert_eq!(args.len(), params.len()); + for (arg, param) in args.iter().zip(params) { + self.make_equal_to(param, *arg); + } + // Remove branch instruction + self.blocks[block.0].insns.pop(); + // Move target instructions into block + let target_insns = std::mem::take(&mut self.blocks[target.0].insns); + self.blocks[block.0].insns.extend(target_insns); + true + } + + /// Clean up linked lists of blocks A -> B -> C into A (with B's and C's instructions). + fn clean_cfg(&mut self) { + // num_in_edges is invariant throughout cleaning the CFG: + // * we don't allocate new blocks + // * blocks that get absorbed are not in RPO anymore + // * blocks pointed to by blocks that get absorbed retain the same number of in-edges + let mut num_in_edges = vec![0; self.blocks.len()]; + for block in self.rpo() { + for &insn in &self.blocks[block.0].insns { + if let Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } | Insn::Jump(target) = self.find(insn) { + num_in_edges[target.target.0] += 1; + } + } + } + let mut changed = false; + loop { + let mut iter_changed = false; + for block in self.rpo() { + // Ignore transient empty blocks + if self.blocks[block.0].insns.is_empty() { continue; } + loop { + let absorbed = self.absorb_dst_block(&num_in_edges, block); + if !absorbed { break; } + iter_changed = true; + } + } + if !iter_changed { break; } + changed = true; + } + if changed { + self.infer_types(); + } + } + + /// Return a list that has entry_block and then jit_entry_blocks + fn entry_blocks(&self) -> Vec<BlockId> { + let mut entry_blocks = self.jit_entry_blocks.clone(); + entry_blocks.insert(0, self.entry_block); + entry_blocks + } + /// Return a traversal of the `Function`'s `BlockId`s in reverse post-order. pub fn rpo(&self) -> Vec<BlockId> { - let mut result = self.po_from(self.entry_block); + let mut result = self.po_from(self.entry_blocks()); result.reverse(); result } - fn po_from(&self, start: BlockId) -> Vec<BlockId> { + fn po_from(&self, starts: Vec<BlockId>) -> Vec<BlockId> { #[derive(PartialEq)] enum Action { VisitEdges, VisitSelf, } let mut result = vec![]; - let mut seen = HashSet::new(); - let mut stack = vec![(start, Action::VisitEdges)]; + let mut seen = BlockSet::with_capacity(self.blocks.len()); + let mut stack: Vec<_> = starts.iter().map(|&start| (start, Action::VisitEdges)).collect(); while let Some((block, action)) = stack.pop() { if action == Action::VisitSelf { result.push(block); @@ -1480,28 +4665,652 @@ impl Function { result } + fn assert_validates(&self) { + if let Err(err) = self.validate() { + eprintln!("Function failed validation."); + eprintln!("Err: {err:?}"); + eprintln!("{}", FunctionPrinter::with_snapshot(self)); + panic!("Aborting..."); + } + } + + /// Helper function to make an Iongraph JSON "instruction". + /// `uses`, `memInputs` and `attributes` are left empty for now, but may be populated + /// in the future. + fn make_iongraph_instr(id: InsnId, inputs: Vec<Json>, opcode: &str, ty: &str) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("opcode", opcode) + .insert("attributes", Json::empty_array()) + .insert("inputs", Json::Array(inputs)) + .insert("uses", Json::empty_array()) + .insert("memInputs", Json::empty_array()) + .insert("type", ty) + .build() + } + + /// Helper function to make an Iongraph JSON "block". + fn make_iongraph_block(id: BlockId, predecessors: Vec<BlockId>, successors: Vec<BlockId>, instructions: Vec<Json>, attributes: Vec<&str>, loop_depth: u32) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("loopDepth", loop_depth) + .insert("attributes", Json::array(attributes)) + .insert("predecessors", Json::array(predecessors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("successors", Json::array(successors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("instructions", Json::array(instructions)) + .build() + } + + /// Helper function to make an Iongraph JSON "function". + /// Note that `lir` is unpopulated right now as ZJIT doesn't use its functionality. + fn make_iongraph_function(pass_name: &str, hir_blocks: Vec<Json>) -> Json { + Json::object() + .insert("name", pass_name) + .insert("mir", Json::object() + .insert("blocks", Json::array(hir_blocks)) + .build() + ) + .insert("lir", Json::object() + .insert("blocks", Json::empty_array()) + .build() + ) + .build() + } + + /// Generate an iongraph JSON pass representation for this function. + pub fn to_iongraph_pass(&self, pass_name: &str) -> Json { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + + let mut hir_blocks = Vec::new(); + let cfi = ControlFlowInfo::new(self); + let dominators = Dominators::new(self); + let loop_info = LoopInfo::new(&cfi, &dominators); + + // Push each block from the iteration in reverse post order to `hir_blocks`. + for block_id in self.rpo() { + // Create the block with instructions. + let block = &self.blocks[block_id.0]; + let predecessors = cfi.predecessors(block_id).collect(); + let successors = cfi.successors(block_id).collect(); + let mut instructions = Vec::new(); + + // Process all instructions (parameters and body instructions). + // Parameters are currently guaranteed to be Parameter instructions, but in the future + // they might be refined to other instruction kinds by the optimizer. + for insn_id in block.params.iter().chain(block.insns.iter()) { + let insn_id = self.union_find.borrow().find_const(*insn_id); + let insn = self.find(insn_id); + + // Snapshots are not serialized, so skip them. + if matches!(insn, Insn::Snapshot {..}) { + continue; + } + + // Instructions with no output or an empty type should have an empty type field. + let type_str = if insn.has_output() { + let insn_type = self.type_of(insn_id); + if insn_type.is_subtype(types::Empty) { + String::new() + } else { + insn_type.print(&ptr_map).to_string() + } + } else { + String::new() + }; + + + let opcode = insn.print(&ptr_map, Some(self.iseq)).to_string(); + + // Traverse the worklist to get inputs for a given instruction. + let mut inputs = VecDeque::new(); + self.worklist_traverse_single_insn(&insn, &mut inputs); + let inputs: Vec<Json> = inputs.into_iter().map(|x| x.0.into()).collect(); + + instructions.push( + Self::make_iongraph_instr( + insn_id, + inputs, + &opcode, + &type_str + ) + ); + } + + let mut attributes = vec![]; + if loop_info.is_back_edge_source(block_id) { + attributes.push("backedge"); + } + if loop_info.is_loop_header(block_id) { + attributes.push("loopheader"); + } + let loop_depth = loop_info.loop_depth(block_id); + + hir_blocks.push(Self::make_iongraph_block( + block_id, + predecessors, + successors, + instructions, + attributes, + loop_depth, + )); + } + + Self::make_iongraph_function(pass_name, hir_blocks) + } + /// Run all the optimization passes we have. pub fn optimize(&mut self) { + let mut passes: Vec<Json> = Vec::new(); + let should_dump = get_option!(dump_hir_iongraph); + + macro_rules! run_pass { + ($name:ident) => { + self.$name(); + #[cfg(debug_assertions)] self.assert_validates(); + if should_dump { + passes.push( + self.to_iongraph_pass(stringify!($name)) + ); + } + } + } + + if should_dump { + passes.push(self.to_iongraph_pass("unoptimized")); + } + // Function is assumed to have types inferred already - self.optimize_direct_sends(); - self.optimize_c_calls(); - self.fold_constants(); - self.eliminate_dead_code(); + run_pass!(type_specialize); + run_pass!(inline); + run_pass!(optimize_getivar); + run_pass!(optimize_c_calls); + run_pass!(fold_constants); + run_pass!(clean_cfg); + run_pass!(eliminate_dead_code); + + if should_dump { + let iseq_name = iseq_get_location(self.iseq, 0); + self.dump_iongraph(&iseq_name, passes); + } + } + /// Dump HIR passed to codegen if specified by options. + pub fn dump_hir(&self) { // Dump HIR after optimization match get_option!(dump_hir_opt) { - Some(DumpHIR::WithoutSnapshot) => println!("HIR:\n{}", FunctionPrinter::without_snapshot(&self)), - Some(DumpHIR::All) => println!("HIR:\n{}", FunctionPrinter::with_snapshot(&self)), - Some(DumpHIR::Debug) => println!("HIR:\n{:#?}", &self), + Some(DumpHIR::WithoutSnapshot) => println!("Optimized HIR:\n{}", FunctionPrinter::without_snapshot(self)), + Some(DumpHIR::All) => println!("Optimized HIR:\n{}", FunctionPrinter::with_snapshot(self)), + Some(DumpHIR::Debug) => println!("Optimized HIR:\n{:#?}", &self), None => {}, } + + if let Some(filename) = &get_option!(dump_hir_graphviz) { + use std::fs::OpenOptions; + use std::io::Write; + let mut file = OpenOptions::new().append(true).open(filename).unwrap(); + writeln!(file, "{}", FunctionGraphvizPrinter::new(self)).unwrap(); + } + } + + pub fn dump_iongraph(&self, function_name: &str, passes: Vec<Json>) { + fn sanitize_for_filename(name: &str) -> String { + name.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' || c == '-' { + c + } else { + '_' + } + }) + .collect() + } + + use std::io::Write; + let dir = format!("/tmp/zjit-iongraph-{}", std::process::id()); + std::fs::create_dir_all(&dir).expect("Unable to create directory."); + let sanitized = sanitize_for_filename(function_name); + let path = format!("{dir}/func_{sanitized}.json"); + let mut file = std::fs::File::create(path).unwrap(); + let json = Json::object() + .insert("name", function_name) + .insert("passes", passes) + .build(); + writeln!(file, "{json}").unwrap(); + } + + /// Validates the following: + /// 1. Basic block jump args match parameter arity. + /// 2. Every terminator must be in the last position. + /// 3. Every block must have a terminator. + fn validate_block_terminators_and_jumps(&self) -> Result<(), ValidationError> { + for block_id in self.rpo() { + let mut block_has_terminator = false; + let insns = &self.blocks[block_id.0].insns; + for (idx, insn_id) in insns.iter().enumerate() { + let insn = self.find(*insn_id); + match &insn { + Insn::Jump(BranchEdge{target, args}) + | Insn::IfTrue { val: _, target: BranchEdge{target, args} } + | Insn::IfFalse { val: _, target: BranchEdge{target, args}} => { + let target_block = &self.blocks[target.0]; + let target_len = target_block.params.len(); + let args_len = args.len(); + if target_len != args_len { + return Err(ValidationError::MismatchedBlockArity(block_id, target_len, args_len)) + } + } + _ => {} + } + if !insn.is_terminator() { + continue; + } + block_has_terminator = true; + if idx != insns.len() - 1 { + return Err(ValidationError::TerminatorNotAtEnd(block_id, *insn_id, idx)); + } + } + if !block_has_terminator { + return Err(ValidationError::BlockHasNoTerminator(block_id)); + } + } + Ok(()) + } + + // This performs a dataflow def-analysis over the entire CFG to detect any + // possibly undefined instruction operands. + fn validate_definite_assignment(&self) -> Result<(), ValidationError> { + // Map of block ID -> InsnSet + // Initialize with all missing values at first, to catch if a jump target points to a + // missing location. + let mut assigned_in = vec![None; self.num_blocks()]; + let rpo = self.rpo(); + // Begin with every block having every variable defined, except for the entry blocks, which + // start with nothing defined. + let entry_blocks = self.entry_blocks(); + for &block in &rpo { + if entry_blocks.contains(&block) { + assigned_in[block.0] = Some(InsnSet::with_capacity(self.insns.len())); + } else { + let mut all_ones = InsnSet::with_capacity(self.insns.len()); + all_ones.insert_all(); + assigned_in[block.0] = Some(all_ones); + } + } + let mut worklist = VecDeque::with_capacity(self.num_blocks()); + worklist.push_back(self.entry_block); + while let Some(block) = worklist.pop_front() { + let mut assigned = assigned_in[block.0].clone().unwrap(); + for ¶m in &self.blocks[block.0].params { + assigned.insert(param); + } + for &insn_id in &self.blocks[block.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + match self.find(insn_id) { + Insn::Jump(target) | Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } => { + let Some(block_in) = assigned_in[target.target.0].as_mut() else { + return Err(ValidationError::JumpTargetNotInRPO(target.target)); + }; + // jump target's block_in was modified, we need to queue the block for processing. + if block_in.intersect_with(&assigned) { + worklist.push_back(target.target); + } + } + insn if insn.has_output() => { + assigned.insert(insn_id); + } + _ => {} + } + } + } + // Check that each instruction's operands are assigned + for &block in &rpo { + let mut assigned = assigned_in[block.0].clone().unwrap(); + for ¶m in &self.blocks[block.0].params { + assigned.insert(param); + } + for &insn_id in &self.blocks[block.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + let mut operands = VecDeque::new(); + let insn = self.find(insn_id); + self.worklist_traverse_single_insn(&insn, &mut operands); + for operand in operands { + if !assigned.get(operand) { + return Err(ValidationError::OperandNotDefined(block, insn_id, operand)); + } + } + if insn.has_output() { + assigned.insert(insn_id); + } + } + } + Ok(()) + } + + /// Checks that each instruction('s representative) appears only once in the CFG. + fn validate_insn_uniqueness(&self) -> Result<(), ValidationError> { + let mut seen = InsnSet::with_capacity(self.insns.len()); + for block_id in self.rpo() { + for &insn_id in &self.blocks[block_id.0].insns { + let insn_id = self.union_find.borrow().find_const(insn_id); + if !seen.insert(insn_id) { + return Err(ValidationError::DuplicateInstruction(block_id, insn_id)); + } + } + } + Ok(()) + } + + fn assert_subtype(&self, user: InsnId, operand: InsnId, expected: Type) -> Result<(), ValidationError> { + let actual = self.type_of(operand); + if !actual.is_subtype(expected) { + return Err(ValidationError::MismatchedOperandType(user, operand, format!("{expected}"), format!("{actual}"))); + } + Ok(()) + } + + fn validate_insn_type(&self, insn_id: InsnId) -> Result<(), ValidationError> { + let insn_id = self.union_find.borrow().find_const(insn_id); + let insn = self.find(insn_id); + match insn { + // Instructions with no InsnId operands (except state) or nothing to assert + Insn::Const { .. } + | Insn::Param + | Insn::PutSpecialObject { .. } + | Insn::LoadField { .. } + | Insn::GetConstantPath { .. } + | Insn::IsBlockGiven + | Insn::GetGlobal { .. } + | Insn::LoadPC + | Insn::LoadEC + | Insn::LoadSelf + | Insn::Snapshot { .. } + | Insn::Jump { .. } + | Insn::EntryPoint { .. } + | Insn::GuardBlockParamProxy { .. } + | Insn::GuardSuperMethodEntry { .. } + | Insn::GetBlockHandler + | Insn::PatchPoint { .. } + | Insn::SideExit { .. } + | Insn::IncrCounter { .. } + | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } + | Insn::GetClassVar { .. } + | Insn::GetSpecialNumber { .. } + | Insn::GetSpecialSymbol { .. } + | Insn::GetLocal { .. } + | Insn::StoreField { .. } => { + Ok(()) + } + // Instructions with 1 Ruby object operand + Insn::Test { val } + | Insn::IsNil { val } + | Insn::IsMethodCfunc { val, .. } + | Insn::GuardShape { val, .. } + | Insn::SetGlobal { val, .. } + | Insn::SetLocal { val, .. } + | Insn::SetClassVar { val, .. } + | Insn::Return { val } + | Insn::Throw { val, .. } + | Insn::ObjToString { val, .. } + | Insn::GuardType { val, .. } + | Insn::GuardTypeNot { val, .. } + | Insn::ToArray { val, .. } + | Insn::ToNewArray { val, .. } + | Insn::Defined { v: val, .. } + | Insn::ObjectAlloc { val, .. } + | Insn::DupArrayInclude { target: val, .. } + | Insn::GetIvar { self_val: val, .. } + | Insn::CCall { recv: val, .. } + | Insn::FixnumBitCheck { val, .. } // TODO (https://github.com/Shopify/ruby/issues/859) this should check Fixnum, but then test_checkkeyword_tests_fixnum_bit fails + | Insn::DefinedIvar { self_val: val, .. } => { + self.assert_subtype(insn_id, val, types::BasicObject) + } + Insn::GuardNotFrozen { recv, .. } | Insn::GuardNotShared { recv, .. } => { + self.assert_subtype(insn_id, recv, types::HeapBasicObject) + } + // Instructions with 2 Ruby object operands + Insn::SetIvar { self_val: left, val: right, .. } + | Insn::NewRange { low: left, high: right, .. } + | Insn::AnyToString { val: left, str: right, .. } + | Insn::WriteBarrier { recv: left, val: right } => { + self.assert_subtype(insn_id, left, types::BasicObject)?; + self.assert_subtype(insn_id, right, types::BasicObject) + } + // Instructions with recv and a Vec of Ruby objects + Insn::SendWithoutBlock { recv, ref args, .. } + | Insn::SendWithoutBlockDirect { recv, ref args, .. } + | Insn::Send { recv, ref args, .. } + | Insn::SendForward { recv, ref args, .. } + | Insn::InvokeSuper { recv, ref args, .. } + | Insn::CCallWithFrame { recv, ref args, .. } + | Insn::CCallVariadic { recv, ref args, .. } + | Insn::InvokeBuiltin { recv, ref args, .. } + | Insn::InvokeProc { recv, ref args, .. } + | Insn::ArrayInclude { target: recv, elements: ref args, .. } => { + self.assert_subtype(insn_id, recv, types::BasicObject)?; + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::ArrayPackBuffer { ref elements, fmt, buffer, .. } => { + self.assert_subtype(insn_id, fmt, types::BasicObject)?; + self.assert_subtype(insn_id, buffer, types::BasicObject)?; + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + // Instructions with a Vec of Ruby objects + Insn::InvokeBlock { ref args, .. } + | Insn::NewArray { elements: ref args, .. } + | Insn::ArrayHash { elements: ref args, .. } + | Insn::ArrayMax { elements: ref args, .. } => { + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::NewHash { ref elements, .. } => { + if elements.len() % 2 != 0 { + return Err(ValidationError::MiscValidationError(insn_id, "NewHash elements length is not even".to_string())); + } + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + Insn::StringConcat { ref strings, .. } + | Insn::ToRegexp { values: ref strings, .. } => { + for &string in strings { + self.assert_subtype(insn_id, string, types::String)?; + } + Ok(()) + } + // Instructions with String operands + Insn::StringCopy { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringIntern { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringAppend { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::String) + } + Insn::StringAppendCodepoint { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::Fixnum) + } + // Instructions with Array operands + Insn::ArrayDup { val, .. } => self.assert_subtype(insn_id, val, types::ArrayExact), + Insn::ArrayExtend { left, right, .. } => { + // TODO(max): Do left and right need to be ArrayExact? + self.assert_subtype(insn_id, left, types::Array)?; + self.assert_subtype(insn_id, right, types::Array) + } + Insn::ArrayPush { array, .. } + | Insn::ArrayPop { array, .. } + | Insn::ArrayLength { array, .. } => { + self.assert_subtype(insn_id, array, types::Array) + } + Insn::ArrayAref { array, index } => { + self.assert_subtype(insn_id, array, types::Array)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + Insn::ArrayAset { array, index, .. } => { + self.assert_subtype(insn_id, array, types::ArrayExact)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + // Instructions with Hash operands + Insn::HashAref { hash, .. } + | Insn::HashAset { hash, .. } => self.assert_subtype(insn_id, hash, types::HashExact), + Insn::HashDup { val, .. } => self.assert_subtype(insn_id, val, types::HashExact), + // Other + Insn::ObjectAllocClass { class, .. } => { + let has_leaf_allocator = unsafe { rb_zjit_class_has_default_allocator(class) } || class_has_leaf_allocator(class); + if !has_leaf_allocator { + return Err(ValidationError::MiscValidationError(insn_id, "ObjectAllocClass must have leaf allocator".to_string())); + } + Ok(()) + } + Insn::IsBitEqual { left, right } + | Insn::IsBitNotEqual { left, right } => { + if self.is_a(left, types::CInt) && self.is_a(right, types::CInt) { + // TODO(max): Check that int sizes match + Ok(()) + } else if self.is_a(left, types::CPtr) && self.is_a(right, types::CPtr) { + Ok(()) + } else if self.is_a(left, types::RubyValue) && self.is_a(right, types::RubyValue) { + Ok(()) + } else { + return Err(ValidationError::MiscValidationError(insn_id, "IsBitEqual can only compare CInt/CInt or RubyValue/RubyValue".to_string())); + } + } + Insn::BoxBool { val } + | Insn::IfTrue { val, .. } + | Insn::IfFalse { val, .. } => { + self.assert_subtype(insn_id, val, types::CBool) + } + Insn::BoxFixnum { val, .. } => self.assert_subtype(insn_id, val, types::CInt64), + Insn::UnboxFixnum { val } => { + self.assert_subtype(insn_id, val, types::Fixnum) + } + Insn::FixnumAref { recv, index } => { + self.assert_subtype(insn_id, recv, types::Fixnum)?; + self.assert_subtype(insn_id, index, types::Fixnum) + } + Insn::FixnumAdd { left, right, .. } + | Insn::FixnumSub { left, right, .. } + | Insn::FixnumMult { left, right, .. } + | Insn::FixnumDiv { left, right, .. } + | Insn::FixnumMod { left, right, .. } + | Insn::FixnumEq { left, right } + | Insn::FixnumNeq { left, right } + | Insn::FixnumLt { left, right } + | Insn::FixnumLe { left, right } + | Insn::FixnumGt { left, right } + | Insn::FixnumGe { left, right } + | Insn::FixnumAnd { left, right } + | Insn::FixnumOr { left, right } + | Insn::FixnumXor { left, right } + | Insn::NewRangeFixnum { low: left, high: right, .. } + => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum) + } + Insn::FixnumLShift { left, right, .. } + | Insn::FixnumRShift { left, right, .. } => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum)?; + let Some(obj) = self.type_of(right).fixnum_value() else { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<a compile-time constant>".into(), "<unknown>".into())); + }; + if obj < 0 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<positive>".into(), format!("{obj}"))); + } + if obj > 63 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<less than 64>".into(), format!("{obj}"))); + } + Ok(()) + } + Insn::GuardBitEquals { val, expected, .. } => { + match expected { + Const::Value(_) => self.assert_subtype(insn_id, val, types::RubyValue), + Const::CInt8(_) => self.assert_subtype(insn_id, val, types::CInt8), + Const::CInt16(_) => self.assert_subtype(insn_id, val, types::CInt16), + Const::CInt32(_) => self.assert_subtype(insn_id, val, types::CInt32), + Const::CInt64(_) => self.assert_subtype(insn_id, val, types::CInt64), + Const::CUInt8(_) => self.assert_subtype(insn_id, val, types::CUInt8), + Const::CUInt16(_) => self.assert_subtype(insn_id, val, types::CUInt16), + Const::CUInt32(_) => self.assert_subtype(insn_id, val, types::CUInt32), + Const::CShape(_) => self.assert_subtype(insn_id, val, types::CShape), + Const::CUInt64(_) => self.assert_subtype(insn_id, val, types::CUInt64), + Const::CBool(_) => self.assert_subtype(insn_id, val, types::CBool), + Const::CDouble(_) => self.assert_subtype(insn_id, val, types::CDouble), + Const::CPtr(_) => self.assert_subtype(insn_id, val, types::CPtr), + } + } + Insn::GuardLess { left, right, .. } + | Insn::GuardGreaterEq { left, right, .. } => { + self.assert_subtype(insn_id, left, types::CInt64)?; + self.assert_subtype(insn_id, right, types::CInt64) + }, + Insn::StringGetbyte { string, index } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::CInt64) + }, + Insn::StringSetbyteFixnum { string, index, value } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::Fixnum)?; + self.assert_subtype(insn_id, value, types::Fixnum) + } + Insn::IsA { val, class } => { + self.assert_subtype(insn_id, val, types::BasicObject)?; + self.assert_subtype(insn_id, class, types::Class) + } + } + } + + /// Check that insn types match the expected types for each instruction. + fn validate_types(&self) -> Result<(), ValidationError> { + for block_id in self.rpo() { + for &insn_id in &self.blocks[block_id.0].insns { + self.validate_insn_type(insn_id)?; + } + } + Ok(()) + } + + /// Run all validation passes we have. + pub fn validate(&self) -> Result<(), ValidationError> { + self.validate_block_terminators_and_jumps()?; + self.validate_definite_assignment()?; + self.validate_insn_uniqueness()?; + self.validate_types()?; + Ok(()) } } impl<'a> std::fmt::Display for FunctionPrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { let fun = &self.fun; - let iseq_name = iseq_name(fun.iseq); + // In tests, there may not be an iseq to get location from. + let iseq_name = if fun.iseq.is_null() { + String::from("<manual>") + } else { + iseq_get_location(fun.iseq, 0) + }; + + // In tests, strip the line number for builtin ISEQs to make tests stable across line changes + let iseq_name = if cfg!(test) && iseq_name.contains("@<internal:") { + iseq_name[..iseq_name.rfind(':').unwrap()].to_string() + } else { + iseq_name + }; writeln!(f, "fn {iseq_name}:")?; for block_id in fun.rpo() { write!(f, "{block_id}(")?; @@ -1519,7 +5328,8 @@ impl<'a> std::fmt::Display for FunctionPrinter<'a> { writeln!(f, "):")?; for insn_id in &fun.blocks[block_id.0].insns { let insn = fun.find(*insn_id); - if !self.display_snapshot && matches!(insn, Insn::Snapshot {..}) { + if !self.display_snapshot_and_tp_patchpoints && + matches!(insn, Insn::Snapshot {..} | Insn::PatchPoint { invariant: Invariant::NoTracePoint, .. }) { continue; } write!(f, " ")?; @@ -1531,13 +5341,94 @@ impl<'a> std::fmt::Display for FunctionPrinter<'a> { write!(f, "{insn_id}:{} = ", insn_type.print(&self.ptr_map))?; } } - writeln!(f, "{}", insn.print(&self.ptr_map))?; + writeln!(f, "{}", insn.print(&self.ptr_map, Some(fun.iseq)))?; + } + } + Ok(()) + } +} + +struct HtmlEncoder<'a, 'b> { + formatter: &'a mut std::fmt::Formatter<'b>, +} + +impl<'a, 'b> std::fmt::Write for HtmlEncoder<'a, 'b> { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + for ch in s.chars() { + match ch { + '<' => self.formatter.write_str("<")?, + '>' => self.formatter.write_str(">")?, + '&' => self.formatter.write_str("&")?, + '"' => self.formatter.write_str(""")?, + '\'' => self.formatter.write_str("'")?, + _ => self.formatter.write_char(ch)?, } } Ok(()) } } +impl<'a> std::fmt::Display for FunctionGraphvizPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + macro_rules! write_encoded { + ($f:ident, $($arg:tt)*) => { + HtmlEncoder { formatter: $f }.write_fmt(format_args!($($arg)*)) + }; + } + use std::fmt::Write; + let fun = &self.fun; + let iseq_name = iseq_get_location(fun.iseq, 0); + write!(f, "digraph G {{ # ")?; + write_encoded!(f, "{iseq_name}")?; + writeln!(f)?; + writeln!(f, "node [shape=plaintext];")?; + writeln!(f, "mode=hier; overlap=false; splines=true;")?; + for block_id in fun.rpo() { + writeln!(f, r#" {block_id} [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">"#)?; + write!(f, r#"<TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">{block_id}("#)?; + if !fun.blocks[block_id.0].params.is_empty() { + let mut sep = ""; + for param in &fun.blocks[block_id.0].params { + write_encoded!(f, "{sep}{param}")?; + let insn_type = fun.type_of(*param); + if !insn_type.is_subtype(types::Empty) { + write_encoded!(f, ":{}", insn_type.print(&self.ptr_map))?; + } + sep = ", "; + } + } + let mut edges = vec![]; + writeln!(f, ") </TD></TR>")?; + for insn_id in &fun.blocks[block_id.0].insns { + let insn_id = fun.union_find.borrow().find_const(*insn_id); + let insn = fun.find(insn_id); + if matches!(insn, Insn::Snapshot {..}) { + continue; + } + write!(f, r#"<TR><TD ALIGN="left" PORT="{insn_id}">"#)?; + if insn.has_output() { + let insn_type = fun.type_of(insn_id); + if insn_type.is_subtype(types::Empty) { + write_encoded!(f, "{insn_id} = ")?; + } else { + write_encoded!(f, "{insn_id}:{} = ", insn_type.print(&self.ptr_map))?; + } + } + if let Insn::Jump(ref target) | Insn::IfTrue { ref target, .. } | Insn::IfFalse { ref target, .. } = insn { + edges.push((insn_id, target.target)); + } + write_encoded!(f, "{}", insn.print(&self.ptr_map, Some(fun.iseq)))?; + writeln!(f, " </TD></TR>")?; + } + writeln!(f, "</TABLE>>];")?; + for (src, dst) in edges { + writeln!(f, " {block_id}:{src} -> {dst}:params:n;")?; + } + } + writeln!(f, "}}") + } +} + #[derive(Debug, Clone, PartialEq)] pub struct FrameState { iseq: IseqPtr, @@ -1550,10 +5441,34 @@ pub struct FrameState { } impl FrameState { - /// Get the opcode for the current instruction - pub fn get_opcode(&self) -> i32 { - unsafe { rb_iseq_opcode_at_pc(self.iseq, self.pc) } + /// Return itself without locals. Useful for side-exiting without spilling locals. + fn without_locals(&self) -> Self { + let mut state = self.clone(); + state.locals.clear(); + state + } + + /// Return itself without stack. Used by leaf calls with GC to reset SP to the base pointer. + pub fn without_stack(&self) -> Self { + let mut state = self.clone(); + state.stack.clear(); + state } + + /// Return itself with send args reordered. Used when kwargs are reordered for callee. + fn with_reordered_args(&self, reordered_args: &[InsnId]) -> Self { + let mut state = self.clone(); + let args_start = state.stack.len() - reordered_args.len(); + state.stack.truncate(args_start); + state.stack.extend_from_slice(reordered_args); + state + } +} + +/// Print adaptor for [`FrameState`]. See [`PtrPrintMap`]. +pub struct FrameStatePrinter<'a> { + inner: &'a FrameState, + ptr_map: &'a PtrPrintMap, } /// Compute the index of a local variable from its slot index @@ -1593,10 +5508,15 @@ impl FrameState { } /// Iterate over all stack slots - pub fn stack(&self) -> Iter<InsnId> { + pub fn stack(&self) -> Iter<'_, InsnId> { self.stack.iter() } + /// Iterate over all local variables + pub fn locals(&self) -> Iter<'_, InsnId> { + self.locals.iter() + } + /// Push a stack operand fn stack_push(&mut self, opnd: InsnId) { self.stack.push(opnd); @@ -1607,6 +5527,16 @@ impl FrameState { self.stack.pop().ok_or_else(|| ParseError::StackUnderflow(self.clone())) } + fn stack_pop_n(&mut self, count: usize) -> Result<Vec<InsnId>, ParseError> { + // Check if we have enough values on the stack + let stack_len = self.stack.len(); + if stack_len < count { + return Err(ParseError::StackUnderflow(self.clone())); + } + + Ok(self.stack.split_off(stack_len - count)) + } + /// Get a stack-top operand fn stack_top(&self) -> Result<InsnId, ParseError> { self.stack.last().ok_or_else(|| ParseError::StackUnderflow(self.clone())).copied() @@ -1634,18 +5564,41 @@ impl FrameState { self.locals[idx] } - fn as_args(&self) -> Vec<InsnId> { - self.locals.iter().chain(self.stack.iter()).map(|op| *op).collect() + fn as_args(&self, self_param: InsnId) -> Vec<InsnId> { + // We're currently passing around the self parameter as a basic block + // argument because the register allocator uses a fixed register based + // on the basic block argument index, which would cause a conflict if + // we reuse an argument from another basic block. + // TODO: Modify the register allocator to allow reusing an argument + // of another basic block. + let mut args = vec![self_param]; + args.extend(self.locals.iter().chain(self.stack.iter()).copied()); + args + } + + /// Get the opcode for the current instruction + pub fn get_opcode(&self) -> i32 { + unsafe { rb_iseq_opcode_at_pc(self.iseq, self.pc) } + } + + pub fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> FrameStatePrinter<'a> { + FrameStatePrinter { inner: self, ptr_map } } } -impl std::fmt::Display for FrameState { +impl Display for FrameStatePrinter<'_> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "FrameState {{ pc: {:?}, stack: ", self.pc)?; - write_vec(f, &self.stack)?; - write!(f, ", locals: ")?; - write_vec(f, &self.locals)?; - write!(f, " }}") + let inner = self.inner; + write!(f, "FrameState {{ pc: {:?}, stack: ", self.ptr_map.map_ptr(inner.pc))?; + write_vec(f, &inner.stack)?; + write!(f, ", locals: [")?; + for (idx, local) in inner.locals.iter().enumerate() { + let name: ID = unsafe { rb_zjit_local_id(inner.iseq, idx.try_into().unwrap()) }; + let name = name.contents_lossy(); + if idx > 0 { write!(f, ", ")?; } + write!(f, "{name}={local}")?; + } + write!(f, "] }}") } } @@ -1659,10 +5612,35 @@ fn insn_idx_at_offset(idx: u32, offset: i64) -> u32 { ((idx as isize) + (offset as isize)) as u32 } -fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec<u32> { +/// List of insn_idx that starts a JIT entry block +pub fn jit_entry_insns(iseq: IseqPtr) -> Vec<u32> { + // TODO(alan): Make an iterator type for this instead of copying all of the opt_table each call + let params = unsafe { iseq.params() }; + let opt_num = params.opt_num; + if opt_num > 0 { + let mut result = vec![]; + + let opt_table = params.opt_table; // `opt_num + 1` entries + for opt_idx in 0..=opt_num as isize { + let insn_idx = unsafe { opt_table.offset(opt_idx).read().as_u32() }; + result.push(insn_idx); + } + result + } else { + vec![0] + } +} + +struct BytecodeInfo { + jump_targets: Vec<u32>, + has_blockiseq: bool, +} + +fn compute_bytecode_info(iseq: *const rb_iseq_t, opt_table: &[u32]) -> BytecodeInfo { let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; let mut insn_idx = 0; - let mut jump_targets = HashSet::new(); + let mut jump_targets: HashSet<u32> = opt_table.iter().copied().collect(); + let mut has_blockiseq = false; while insn_idx < iseq_size { // Get the current pc and opcode let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; @@ -1686,58 +5664,60 @@ fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec<u32> { jump_targets.insert(insn_idx); } } + YARVINSN_send | YARVINSN_invokesuper => { + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + if !blockiseq.is_null() { + has_blockiseq = true; + } + } _ => {} } } let mut result = jump_targets.into_iter().collect::<Vec<_>>(); result.sort(); - result + BytecodeInfo { jump_targets: result, has_blockiseq } } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum CallType { Splat, - BlockArg, Kwarg, - KwSplat, Tailcall, - Super, - Zsuper, - OptSend, - KwSplatMut, - SplatMut, - Forwarding, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ParseError { StackUnderflow(FrameState), - UnknownOpcode(String), - UnknownNewArraySend(String), - UnhandledCallType(CallType), + MalformedIseq(u32), // insn_idx into iseq_encoded + Validation(ValidationError), + NotAllowed, } /// Return the number of locals in the current ISEQ (includes parameters) fn num_locals(iseq: *const rb_iseq_t) -> usize { - (unsafe { get_iseq_body_local_table_size(iseq) }).as_usize() + (unsafe { get_iseq_body_local_table_size(iseq) }).to_usize() } /// If we can't handle the type of send (yet), bail out. -fn filter_translatable_calls(flag: u32) -> Result<(), ParseError> { - if (flag & VM_CALL_KW_SPLAT_MUT) != 0 { return Err(ParseError::UnhandledCallType(CallType::KwSplatMut)); } - if (flag & VM_CALL_ARGS_SPLAT_MUT) != 0 { return Err(ParseError::UnhandledCallType(CallType::SplatMut)); } - if (flag & VM_CALL_ARGS_SPLAT) != 0 { return Err(ParseError::UnhandledCallType(CallType::Splat)); } - if (flag & VM_CALL_KW_SPLAT) != 0 { return Err(ParseError::UnhandledCallType(CallType::KwSplat)); } - if (flag & VM_CALL_ARGS_BLOCKARG) != 0 { return Err(ParseError::UnhandledCallType(CallType::BlockArg)); } - if (flag & VM_CALL_KWARG) != 0 { return Err(ParseError::UnhandledCallType(CallType::Kwarg)); } - if (flag & VM_CALL_TAILCALL) != 0 { return Err(ParseError::UnhandledCallType(CallType::Tailcall)); } - if (flag & VM_CALL_SUPER) != 0 { return Err(ParseError::UnhandledCallType(CallType::Super)); } - if (flag & VM_CALL_ZSUPER) != 0 { return Err(ParseError::UnhandledCallType(CallType::Zsuper)); } - if (flag & VM_CALL_OPT_SEND) != 0 { return Err(ParseError::UnhandledCallType(CallType::OptSend)); } - if (flag & VM_CALL_FORWARDING) != 0 { return Err(ParseError::UnhandledCallType(CallType::Forwarding)); } +fn unhandled_call_type(flags: u32) -> Result<(), CallType> { + if (flags & VM_CALL_TAILCALL) != 0 { return Err(CallType::Tailcall); } Ok(()) } +/// If a given call to a c func uses overly complex arguments, then we won't specialize. +fn unspecializable_c_call_type(flags: u32) -> bool { + ((flags & VM_CALL_KWARG) != 0) || + unspecializable_call_type(flags) +} + +/// If a given call uses overly complex arguments, then we won't specialize. +fn unspecializable_call_type(flags: u32) -> bool { + ((flags & VM_CALL_ARGS_SPLAT) != 0) || + ((flags & VM_CALL_KW_SPLAT) != 0) || + ((flags & VM_CALL_ARGS_BLOCKARG) != 0) || + ((flags & VM_CALL_FORWARDING) != 0) +} + /// We have IseqPayload, which keeps track of HIR Types in the interpreter, but this is not useful /// or correct to query from inside the optimizer. Instead, ProfileOracle provides an API to look /// up profiled type information by HIR InsnId at a given ISEQ instruction. @@ -1748,7 +5728,7 @@ struct ProfileOracle { /// instruction index. At a given ISEQ instruction, the interpreter has profiled the stack /// operands to a given ISEQ instruction, and this list of pairs of (InsnId, Type) map that /// profiling information into HIR instructions. - types: HashMap<usize, Vec<(InsnId, Type)>>, + types: HashMap<usize, Vec<(InsnId, TypeDistributionSummary)>>, } impl ProfileOracle { @@ -1759,83 +5739,120 @@ impl ProfileOracle { /// Map the interpreter-recorded types of the stack onto the HIR operands on our compile-time virtual stack fn profile_stack(&mut self, state: &FrameState) { let iseq_insn_idx = state.insn_idx; - let Some(operand_types) = self.payload.get_operand_types(iseq_insn_idx) else { return }; - let entry = self.types.entry(iseq_insn_idx).or_insert_with(|| vec![]); + let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; + let entry = self.types.entry(iseq_insn_idx).or_default(); // operand_types is always going to be <= stack size (otherwise it would have an underflow // at run-time) so use that to drive iteration. - for (idx, &insn_type) in operand_types.iter().rev().enumerate() { + for (idx, insn_type_distribution) in operand_types.iter().rev().enumerate() { let insn = state.stack_topn(idx).expect("Unexpected stack underflow in profiling"); - entry.push((insn, insn_type)) + entry.push((insn, TypeDistributionSummary::new(insn_type_distribution))) + } + } + + /// Map the interpreter-recorded types of self onto the HIR self + fn profile_self(&mut self, state: &FrameState, self_param: InsnId) { + let iseq_insn_idx = state.insn_idx; + let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; + let entry = self.types.entry(iseq_insn_idx).or_default(); + if operand_types.is_empty() { + return; } + let self_type_distribution = &operand_types[0]; + entry.push((self_param, TypeDistributionSummary::new(self_type_distribution))) } } +fn invalidates_locals(opcode: u32, operands: *const VALUE) -> bool { + match opcode { + // Control-flow is non-leaf in the interpreter because it can execute arbitrary code on + // interrupt. But in the JIT, we side-exit if there is a pending interrupt. + YARVINSN_jump + | YARVINSN_branchunless + | YARVINSN_branchif + | YARVINSN_branchnil + | YARVINSN_leave => false, + // TODO(max): Read the invokebuiltin target from operands and determine if it's leaf + _ => unsafe { !rb_zjit_insn_leaf(opcode as i32, operands) } + } +} + +/// The index of the self parameter in the HIR function +pub const SELF_PARAM_IDX: usize = 0; + /// Compile ISEQ into High-level IR pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { + if !ZJITState::can_compile_iseq(iseq) { + return Err(ParseError::NotAllowed); + } let payload = get_or_create_iseq_payload(iseq); let mut profiles = ProfileOracle::new(payload); let mut fun = Function::new(iseq); + // Compute a map of PC->Block by finding jump targets - let jump_targets = compute_jump_targets(iseq); + let jit_entry_insns = jit_entry_insns(iseq); + let BytecodeInfo { jump_targets, has_blockiseq } = compute_bytecode_info(iseq, &jit_entry_insns); + + // Make all empty basic blocks. The ordering of the BBs matters for getting fallthrough jumps + // in good places, but it's not necessary for correctness. TODO: Higher quality scheduling during lowering. let mut insn_idx_to_block = HashMap::new(); + // Make blocks for optionals first, and put them right next to their JIT entrypoint + for insn_idx in jit_entry_insns.iter().copied() { + let jit_entry_block = fun.new_block(insn_idx); + fun.jit_entry_blocks.push(jit_entry_block); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Make blocks for the rest of the jump targets for insn_idx in jump_targets { - if insn_idx == 0 { - todo!("Separate entry block for param/self/..."); - } - insn_idx_to_block.insert(insn_idx, fun.new_block()); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); } + // Done, drop `mut`. + let insn_idx_to_block = insn_idx_to_block; - // Iteratively fill out basic blocks using a queue - // TODO(max): Basic block arguments at edges - let mut queue = std::collections::VecDeque::new(); - // Index of the rest parameter for comparison below - let rest_param_idx = if !iseq.is_null() && unsafe { get_iseq_flags_has_rest(iseq) } { - let opt_num = unsafe { get_iseq_body_param_opt_num(iseq) }; - let lead_num = unsafe { get_iseq_body_param_lead_num(iseq) }; - opt_num + lead_num - } else { - -1 - }; - // The HIR function will have the same number of parameter as the iseq so - // we properly handle calls from the interpreter. Roughly speaking, each - // item between commas in the source increase the parameter count by one, - // regardless of parameter kind. - let mut entry_state = FrameState::new(iseq); - for idx in 0..num_locals(iseq) { - if idx < unsafe { get_iseq_body_param_size(iseq) }.as_usize() { - entry_state.locals.push(fun.push_insn(fun.entry_block, Insn::Param { idx })); - } else { - entry_state.locals.push(fun.push_insn(fun.entry_block, Insn::Const { val: Const::Value(Qnil) })); - } + // Compile an entry_block for the interpreter + compile_entry_block(&mut fun, jit_entry_insns.as_slice(), &insn_idx_to_block); - let mut param_type = types::BasicObject; - // Rest parameters are always ArrayExact - if let Ok(true) = c_int::try_from(idx).map(|idx| idx == rest_param_idx) { - param_type = types::ArrayExact; - } - fun.param_types.push(param_type); + // Compile all JIT-to-JIT entry blocks + for (jit_entry_idx, insn_idx) in jit_entry_insns.iter().enumerate() { + let target_block = insn_idx_to_block.get(insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + compile_jit_entry_block(&mut fun, jit_entry_idx, target_block); } - queue.push_back((entry_state, fun.entry_block, /*insn_idx=*/0_u32)); - let mut visited = HashSet::new(); + // Check if the EP is escaped for the ISEQ from the beginning. We give up + // optimizing locals in that case because they're shared with other frames. + let ep_escaped = iseq_escapes_ep(iseq); + + // Iteratively fill out basic blocks using a queue. + // TODO(max): Basic block arguments at edges + let mut queue = VecDeque::new(); + for &insn_idx in jit_entry_insns.iter() { + queue.push_back((FrameState::new(iseq), insn_idx_to_block[&insn_idx], /*insn_idx=*/insn_idx, /*local_inval=*/false)); + } + // Keep compiling blocks until the queue becomes empty + let mut visited = HashSet::new(); let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; - while let Some((incoming_state, block, mut insn_idx)) = queue.pop_front() { + while let Some((incoming_state, block, mut insn_idx, mut local_inval)) = queue.pop_front() { + // Compile each block only once if visited.contains(&block) { continue; } visited.insert(block); - let mut state = if insn_idx == 0 { incoming_state.clone() } else { + + // Load basic block params first + let self_param = fun.push_insn(block, Insn::Param); + let mut state = { let mut result = FrameState::new(iseq); - let mut idx = 0; - for _ in 0..incoming_state.locals.len() { - result.locals.push(fun.push_insn(block, Insn::Param { idx })); - idx += 1; + let local_size = if jit_entry_insns.contains(&insn_idx) { num_locals(iseq) } else { incoming_state.locals.len() }; + for _ in 0..local_size { + result.locals.push(fun.push_insn(block, Insn::Param)); } for _ in incoming_state.stack { - result.stack.push(fun.push_insn(block, Insn::Param { idx })); - idx += 1; + result.stack.push(fun.push_insn(block, Insn::Param)); } result }; + // Start the block off with a Snapshot so that if we need to insert a new Guard later on // and we don't have a Snapshot handy, we can just iterate backward (at the earliest, to // the beginning of the block). @@ -1846,12 +5863,72 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; state.pc = pc; let exit_state = state.clone(); - profiles.profile_stack(&exit_state); // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } .try_into() .unwrap(); + + // If TracePoint has been enabled after we have collected profiles, we'll see + // trace_getinstancevariable in the ISEQ. We have to treat it like getinstancevariable + // for profiling purposes: there is no operand on the stack to look up; we have + // profiled cfp->self. + if opcode == YARVINSN_getinstancevariable || opcode == YARVINSN_trace_getinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_setinstancevariable || opcode == YARVINSN_trace_setinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_definedivar || opcode == YARVINSN_trace_definedivar { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_invokeblock || opcode == YARVINSN_trace_invokeblock { + if get_option!(stats) { + let iseq_insn_idx = exit_state.insn_idx; + if let Some(operand_types) = profiles.payload.profile.get_operand_types(iseq_insn_idx) { + if let [self_type_distribution] = &operand_types[..] { + let summary = TypeDistributionSummary::new(&self_type_distribution); + if summary.is_monomorphic() { + let obj = summary.bucket(0).class(); + if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 } { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_iseq)); + } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 } { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_ifunc)); + } else { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_monomorphic_other)); + } + } else if summary.is_skewed_polymorphic() || summary.is_polymorphic() { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_polymorphic)); + } else if summary.is_skewed_megamorphic() || summary.is_megamorphic() { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_megamorphic)); + } else { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_no_profiles)); + } + } else { + fun.push_insn(block, Insn::IncrCounter(Counter::invokeblock_handler_no_profiles)); + } + } + } + } else { + profiles.profile_stack(&exit_state); + } + + // Flag a future getlocal/setlocal to add a patch point if this instruction is not leaf. + if invalidates_locals(opcode, unsafe { pc.offset(1) }) { + local_inval = true; + } + + // We add NoTracePoint patch points before every instruction that could be affected by TracePoint. + // This ensures that if TracePoint is enabled, we can exit the generated code as fast as possible. + unsafe extern "C" { + fn rb_iseq_event_flags(iseq: IseqPtr, pos: usize) -> rb_event_flag_t; + } + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.clone() }); + if unsafe { rb_iseq_event_flags(iseq, insn_idx as usize) } != 0 { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state: exit_id }); + } + + // Increment zjit_insn_count for each YARV instruction if --zjit-stats is enabled. + if get_option!(stats) { + fun.push_insn(block, Insn::IncrCounter(Counter::zjit_insn_count)); + } // Move to the next instruction to compile insn_idx += insn_len(opcode as usize); @@ -1859,54 +5936,160 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { YARVINSN_nop => {}, YARVINSN_putnil => { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) })); }, YARVINSN_putobject => { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) })); }, - YARVINSN_putstring | YARVINSN_putchilledstring => { - // TODO(max): Do something different for chilled string + YARVINSN_putspecialobject => { + let value_type = SpecialObjectType::from(get_arg(pc, 0).as_u32()); + let insn = if value_type == SpecialObjectType::VMCore { + Insn::Const { val: Const::Value(unsafe { rb_mRubyVMFrozenCore }) } + } else { + Insn::PutSpecialObject { value_type } + }; + state.stack_push(fun.push_insn(block, insn)); + } + YARVINSN_putstring => { let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let insn_id = fun.push_insn(block, Insn::StringCopy { val }); + let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: false, state: exit_id }); state.stack_push(insn_id); } - YARVINSN_putself => { state.stack_push(fun.push_insn(block, Insn::PutSelf)); } + YARVINSN_putchilledstring => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: true, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_putself => { state.stack_push(self_param); } YARVINSN_intern => { let val = state.stack_pop()?; - let insn_id = fun.push_insn(block, Insn::StringIntern { val }); + let insn_id = fun.push_insn(block, Insn::StringIntern { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_concatstrings => { + let count = get_arg(pc, 0).as_u32(); + let strings = state.stack_pop_n(count as usize)?; + let insn_id = fun.push_insn(block, Insn::StringConcat { strings, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_toregexp => { + // First arg contains the options (multiline, extended, ignorecase) used to create the regexp + let opt = get_arg(pc, 0).as_usize(); + let count = get_arg(pc, 1).as_usize(); + let values = state.stack_pop_n(count)?; + let insn_id = fun.push_insn(block, Insn::ToRegexp { opt, values, state: exit_id }); state.stack_push(insn_id); } YARVINSN_newarray => { let count = get_arg(pc, 0).as_usize(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let mut elements = vec![]; - for _ in 0..count { - elements.push(state.stack_pop()?); - } - elements.reverse(); + let elements = state.stack_pop_n(count)?; state.stack_push(fun.push_insn(block, Insn::NewArray { elements, state: exit_id })); } YARVINSN_opt_newarray_send => { let count = get_arg(pc, 0).as_usize(); let method = get_arg(pc, 1).as_u32(); - let mut elements = vec![]; - for _ in 0..count { - elements.push(state.stack_pop()?); - } - elements.reverse(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let (bop, insn) = match method { - VM_OPT_NEWARRAY_SEND_MAX => (BOP_MAX, Insn::ArrayMax { elements, state: exit_id }), - VM_OPT_NEWARRAY_SEND_MIN => return Err(ParseError::UnknownNewArraySend("min".into())), - VM_OPT_NEWARRAY_SEND_HASH => return Err(ParseError::UnknownNewArraySend("hash".into())), - VM_OPT_NEWARRAY_SEND_PACK => return Err(ParseError::UnknownNewArraySend("pack".into())), - VM_OPT_NEWARRAY_SEND_PACK_BUFFER => return Err(ParseError::UnknownNewArraySend("pack_buffer".into())), - _ => return Err(ParseError::UnknownNewArraySend(format!("{method}"))), + VM_OPT_NEWARRAY_SEND_MAX => { + let elements = state.stack_pop_n(count)?; + (BOP_MAX, Insn::ArrayMax { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_HASH => { + let elements = state.stack_pop_n(count)?; + (BOP_HASH, Insn::ArrayHash { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_INCLUDE_P => { + let target = state.stack_pop()?; + let elements = state.stack_pop_n(count - 1)?; + (BOP_INCLUDE_P, Insn::ArrayInclude { elements, target, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => { + let buffer = state.stack_pop()?; + let fmt = state.stack_pop()?; + let elements = state.stack_pop_n(count - 2)?; + (BOP_PACK, Insn::ArrayPackBuffer { elements, fmt, buffer, state: exit_id }) + } + _ => { + // Unknown opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledNewarraySend(method) }); + break; // End the block + } }; - fun.push_insn(block, Insn::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop })); + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }) }); + break; // End the block + } + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); state.stack_push(fun.push_insn(block, insn)); } YARVINSN_duparray => { let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let insn_id = fun.push_insn(block, Insn::ArrayDup { val, state: exit_id }); state.stack_push(insn_id); } + YARVINSN_opt_duparray_send => { + let ary = get_arg(pc, 0); + let method_id = get_arg(pc, 1).as_u64(); + let argc = get_arg(pc, 2).as_usize(); + if argc != 1 { + break; + } + let target = state.stack_pop()?; + let bop = match method_id { + x if x == ID!(include_p).0 => BOP_INCLUDE_P, + _ => { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDuparraySend(method_id) }); + break; + }, + }; + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }) }); + break; + } + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); + let insn_id = fun.push_insn(block, Insn::DupArrayInclude { ary, target, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_newhash => { + let count = get_arg(pc, 0).as_usize(); + assert!(count % 2 == 0, "newhash count should be even"); + let mut elements = vec![]; + for _ in 0..(count/2) { + let value = state.stack_pop()?; + let key = state.stack_pop()?; + elements.push(value); + elements.push(key); + } + elements.reverse(); + state.stack_push(fun.push_insn(block, Insn::NewHash { elements, state: exit_id })); + } + YARVINSN_duphash => { + let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + let insn_id = fun.push_insn(block, Insn::HashDup { val, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_splatarray => { + let flag = get_arg(pc, 0); + let result_must_be_mutable = flag.test(); + let val = state.stack_pop()?; + let obj = if result_must_be_mutable { + fun.push_insn(block, Insn::ToNewArray { val, state: exit_id }) + } else { + fun.push_insn(block, Insn::ToArray { val, state: exit_id }) + }; + state.stack_push(obj); + } + YARVINSN_concattoarray => { + let right = state.stack_pop()?; + let left = state.stack_pop()?; + let right_array = fun.push_insn(block, Insn::ToArray { val: right, state: exit_id }); + fun.push_insn(block, Insn::ArrayExtend { left, right: right_array, state: exit_id }); + state.stack_push(left); + } + YARVINSN_pushtoarray => { + let count = get_arg(pc, 0).as_usize(); + let vals = state.stack_pop_n(count)?; + let array = state.stack_pop()?; + for val in vals.into_iter() { + fun.push_insn(block, Insn::ArrayPush { array, val, state: exit_id }); + } + state.stack_push(array); + } YARVINSN_putobject_INT2FIX_0_ => { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(0)) })); } @@ -1914,76 +6097,194 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(1)) })); } YARVINSN_defined => { + // (rb_num_t op_type, VALUE obj, VALUE pushval) let op_type = get_arg(pc, 0).as_usize(); - let obj = get_arg(pc, 0); - let pushval = get_arg(pc, 0); + let obj = get_arg(pc, 1); + let pushval = get_arg(pc, 2); let v = state.stack_pop()?; - state.stack_push(fun.push_insn(block, Insn::Defined { op_type, obj, pushval, v })); + state.stack_push(fun.push_insn(block, Insn::Defined { op_type, obj, pushval, v, state: exit_id })); + } + YARVINSN_definedivar => { + // (ID id, IVC ic, VALUE pushval) + let id = ID(get_arg(pc, 0).as_u64()); + let pushval = get_arg(pc, 2); + state.stack_push(fun.push_insn(block, Insn::DefinedIvar { self_val: self_param, id, pushval, state: exit_id })); + } + YARVINSN_checkkeyword => { + // When a keyword is unspecified past index 32, a hash will be used instead. + // This can only happen in iseqs taking more than 32 keywords. + // In this case, we side exit to the interpreter. + if unsafe {(*rb_get_iseq_body_param_keyword(iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap()} { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyKeywordParameters }); + break; + } + let ep_offset = get_arg(pc, 0).as_u32(); + let index = get_arg(pc, 1).as_u64(); + let index: u8 = index.try_into().map_err(|_| ParseError::MalformedIseq(insn_idx))?; + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.stack_push(fun.push_insn(block, Insn::FixnumBitCheck { val, index })); } YARVINSN_opt_getconstant_path => { let ic = get_arg(pc, 0).as_ptr(); - state.stack_push(fun.push_insn(block, Insn::GetConstantPath { ic })); + let snapshot = fun.push_insn(block, Insn::Snapshot { state: exit_state }); + state.stack_push(fun.push_insn(block, Insn::GetConstantPath { ic, state: snapshot })); } YARVINSN_branchunless => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); let offset = get_arg(pc, 0).as_i64(); let val = state.stack_pop()?; let test_id = fun.push_insn(block, Insn::Test { val }); - // TODO(max): Check interrupts let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; let _branch_id = fun.push_insn(block, Insn::IfFalse { val: test_id, - target: BranchEdge { target, args: state.as_args() } + target: BranchEdge { target, args: state.as_args(self_param) } }); - queue.push_back((state.clone(), target, target_idx)); + queue.push_back((state.clone(), target, target_idx, local_inval)); } YARVINSN_branchif => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); let offset = get_arg(pc, 0).as_i64(); let val = state.stack_pop()?; let test_id = fun.push_insn(block, Insn::Test { val }); - // TODO(max): Check interrupts let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; let _branch_id = fun.push_insn(block, Insn::IfTrue { val: test_id, - target: BranchEdge { target, args: state.as_args() } + target: BranchEdge { target, args: state.as_args(self_param) } }); - queue.push_back((state.clone(), target, target_idx)); + queue.push_back((state.clone(), target, target_idx, local_inval)); } - YARVINSN_opt_new => { - let offset = get_arg(pc, 1).as_i64(); - // TODO(max): Check interrupts + YARVINSN_branchnil => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + let offset = get_arg(pc, 0).as_i64(); + let val = state.stack_pop()?; + let test_id = fun.push_insn(block, Insn::IsNil { val }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; - // Skip the fast-path and go straight to the fallback code. We will let the - // optimizer take care of the converting Class#new->alloc+initialize instead. - fun.push_insn(block, Insn::Jump(BranchEdge { target, args: state.as_args() })); - queue.push_back((state.clone(), target, target_idx)); - break; // Don't enqueue the next block as a successor + let _branch_id = fun.push_insn(block, Insn::IfTrue { + val: test_id, + target: BranchEdge { target, args: state.as_args(self_param) } + }); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_opt_case_dispatch => { + // TODO: Some keys are visible at compile time, so in the future we can + // compile jump targets for certain cases + // Pop the key from the stack and fallback to the === branches for now + state.stack_pop()?; + } + YARVINSN_opt_new => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let dst = get_arg(pc, 1).as_i64(); + + // Check if #new resolves to rb_class_new_instance_pass_kw. + // TODO: Guard on a profiled class and add a patch point for #new redefinition + let argc = unsafe { vm_ci_argc((*cd).ci) } as usize; + let val = state.stack_topn(argc)?; + let test_id = fun.push_insn(block, Insn::IsMethodCfunc { val, cd, cfunc: rb_class_new_instance_pass_kw as *const u8, state: exit_id }); + + // Jump to the fallback block if it's not the expected function. + // Skip CheckInterrupts since the #new call will do it very soon anyway. + let target_idx = insn_idx_at_offset(insn_idx, dst); + let target = insn_idx_to_block[&target_idx]; + let _branch_id = fun.push_insn(block, Insn::IfFalse { + val: test_id, + target: BranchEdge { target, args: state.as_args(self_param) } + }); + queue.push_back((state.clone(), target, target_idx, local_inval)); + + // Move on to the fast path + let insn_id = fun.push_insn(block, Insn::ObjectAlloc { val, state: exit_id }); + state.stack_setn(argc, insn_id); + state.stack_setn(argc + 1, insn_id); } YARVINSN_jump => { let offset = get_arg(pc, 0).as_i64(); - // TODO(max): Check interrupts + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; let _branch_id = fun.push_insn(block, Insn::Jump( - BranchEdge { target, args: state.as_args() } + BranchEdge { target, args: state.as_args(self_param) } )); - queue.push_back((state.clone(), target, target_idx)); + queue.push_back((state.clone(), target, target_idx, local_inval)); break; // Don't enqueue the next block as a successor } YARVINSN_getlocal_WC_0 => { let ep_offset = get_arg(pc, 0).as_u32(); - let val = state.getlocal(ep_offset); - state.stack_push(val); + if !local_inval { + // The FrameState is the source of truth for locals until invalidated. + // In case of JIT-to-JIT send locals might never end up in EP memory. + let val = state.getlocal(ep_offset); + state.stack_push(val); + } else if ep_escaped || has_blockiseq { // TODO: figure out how to drop has_blockiseq here + // Read the local using EP + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); // remember the result to spill on side-exits + state.stack_push(val); + } else { + assert!(local_inval); // if check above + // There has been some non-leaf call since JIT entry or the last patch point, + // so add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + + // Read the local from FrameState + let val = state.getlocal(ep_offset); + state.stack_push(val); + } } YARVINSN_setlocal_WC_0 => { let ep_offset = get_arg(pc, 0).as_u32(); let val = state.stack_pop()?; + if ep_escaped || has_blockiseq { // TODO: figure out how to drop has_blockiseq here + // Write the local using EP + fun.push_insn(block, Insn::SetLocal { val, ep_offset, level: 0 }); + } else if local_inval { + // If there has been any non-leaf call since JIT entry or the last patch point, + // add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + } + // Write the local into FrameState state.setlocal(ep_offset, val); } + YARVINSN_getlocal_WC_1 => { + let ep_offset = get_arg(pc, 0).as_u32(); + state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level: 1, use_sp: false, rest_param: false })); + } + YARVINSN_setlocal_WC_1 => { + let ep_offset = get_arg(pc, 0).as_u32(); + fun.push_insn(block, Insn::SetLocal { val: state.stack_pop()?, ep_offset, level: 1 }); + } + YARVINSN_getlocal => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level, use_sp: false, rest_param: false })); + } + YARVINSN_setlocal => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + fun.push_insn(block, Insn::SetLocal { val: state.stack_pop()?, ep_offset, level }); + } + YARVINSN_getblockparamproxy => { + let level = get_arg(pc, 1).as_u32(); + fun.push_insn(block, Insn::GuardBlockParamProxy { level, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) })); + } YARVINSN_pop => { state.stack_pop()?; } YARVINSN_dup => { state.stack_push(state.stack_top()?); } + YARVINSN_dupn => { + // Duplicate the top N element of the stack. As we push, n-1 naturally + // points higher in the original stack. + let n = get_arg(pc, 0).as_usize(); + for _ in 0..n { + state.stack_push(state.stack_topn(n-1)?); + } + } YARVINSN_swap => { let right = state.stack_pop()?; let left = state.stack_pop()?; @@ -2007,36 +6308,85 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { n -= 1; } } - YARVINSN_opt_neq => { // NB: opt_neq has two cd; get_arg(0) is for eq and get_arg(1) is for neq let cd: *const rb_call_data = get_arg(pc, 1).as_ptr(); let call_info = unsafe { rb_get_call_data_ci(cd) }; - filter_translatable_calls(unsafe { rb_vm_ci_flag(call_info) })?; - let argc = unsafe { vm_ci_argc((*cd).ci) }; - - - let method_name = unsafe { - let mid = rb_vm_ci_mid(call_info); - mid.contents_lossy().into_owned() - }; - let mut args = vec![]; - for _ in 0..argc { - args.push(state.stack_pop()?); + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block } - args.reverse(); + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let args = state.stack_pop_n(argc as usize)?; let recv = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let send = fun.push_insn(block, Insn::SendWithoutBlock { self_val: recv, call_info: CallInfo { method_name }, cd, args, state: exit_id }); + let send = fun.push_insn(block, Insn::SendWithoutBlock { recv, cd, args, state: exit_id, reason: Uncategorized(opcode) }); state.stack_push(send); } - + YARVINSN_opt_hash_freeze => { + let klass = HASH_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } + YARVINSN_opt_ary_freeze => { + let klass = ARRAY_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } + YARVINSN_opt_str_freeze => { + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } + YARVINSN_opt_str_uminus => { + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_UMINUS; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }) }); + break; // End the block + } + } YARVINSN_leave => { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); fun.push_insn(block, Insn::Return { val: state.stack_pop()? }); break; // Don't enqueue the next block as a successor } + YARVINSN_throw => { + fun.push_insn(block, Insn::Throw { throw_state: get_arg(pc, 0).as_u32(), val: state.stack_pop()?, state: exit_id }); + break; // Don't enqueue the next block as a successor + } + // These are opt_send_without_block and all the opt_* instructions + // specialized to a certain method that could also be serviced + // using the general send implementation. The optimizer start from + // a general send for all of these later in the pipeline. YARVINSN_opt_nil_p | YARVINSN_opt_plus | YARVINSN_opt_minus | @@ -2052,75 +6402,721 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { YARVINSN_opt_aset | YARVINSN_opt_length | YARVINSN_opt_size | + YARVINSN_opt_aref | + YARVINSN_opt_empty_p | + YARVINSN_opt_succ | + YARVINSN_opt_and | + YARVINSN_opt_or | + YARVINSN_opt_not | + YARVINSN_opt_regexpmatch2 | YARVINSN_opt_send_without_block => { let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); let call_info = unsafe { rb_get_call_data_ci(cd) }; - filter_translatable_calls(unsafe { rb_vm_ci_flag(call_info) })?; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } let argc = unsafe { vm_ci_argc((*cd).ci) }; - - let method_name = unsafe { - let mid = rb_vm_ci_mid(call_info); - mid.contents_lossy().into_owned() - }; - let mut args = vec![]; - for _ in 0..argc { - args.push(state.stack_pop()?); + let args = state.stack_pop_n(argc as usize)?; + let recv = state.stack_pop()?; + let send = fun.push_insn(block, Insn::SendWithoutBlock { recv, cd, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send); + } + YARVINSN_send => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block } - args.reverse(); + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; let recv = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let send = fun.push_insn(block, Insn::SendWithoutBlock { self_val: recv, call_info: CallInfo { method_name }, cd, args, state: exit_id }); + let send = fun.push_insn(block, Insn::Send { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); state.stack_push(send); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + // TODO: We could use `use_sp: true` with PatchPoint + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); + } + } } - YARVINSN_send => { + YARVINSN_sendforward => { let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); let call_info = unsafe { rb_get_call_data_ci(cd) }; - filter_translatable_calls(unsafe { rb_vm_ci_flag(call_info) })?; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + let forwarding = (flags & VM_CALL_FORWARDING) != 0; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } let argc = unsafe { vm_ci_argc((*cd).ci) }; - let method_name = unsafe { - let mid = rb_vm_ci_mid(call_info); - mid.contents_lossy().into_owned() - }; + let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; + let recv = state.stack_pop()?; + let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(send_forward); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + // TODO: We could use `use_sp: true` with PatchPoint + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); + } + } + } + YARVINSN_invokesuper => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; + let recv = state.stack_pop()?; + let blockiseq: IseqPtr = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::InvokeSuper { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(result); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + // TODO: We could use `use_sp: true` with PatchPoint + let val = fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0, use_sp: false, rest_param: false }); + state.setlocal(ep_offset, val); + } + } + } + YARVINSN_invokeblock => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type) }); + break; // End the block + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; + let result = fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(result); + } + YARVINSN_getglobal => { + let id = ID(get_arg(pc, 0).as_u64()); + let result = fun.push_insn(block, Insn::GetGlobal { id, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setglobal => { + let id = ID(get_arg(pc, 0).as_u64()); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetGlobal { id, val, state: exit_id }); + } + YARVINSN_getinstancevariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + // ic is in arg 1 + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_getivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_getivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block + } + let result = fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setinstancevariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_setivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_setivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block + } + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, ic, val, state: exit_id }); + } + YARVINSN_getclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::GetClassVar { id, ic, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetClassVar { id, val, ic, state: exit_id }); + } + YARVINSN_opt_reverse => { + // Reverse the order of the top N stack items. + let n = get_arg(pc, 0).as_usize(); + for i in 0..n/2 { + let bottom = state.stack_topn(n - 1 - i)?; + let top = state.stack_topn(i)?; + state.stack_setn(i, bottom); + state.stack_setn(n - 1 - i, top); + } + } + YARVINSN_newrange => { + let flag = RangeType::from(get_arg(pc, 0).as_u32()); + let high = state.stack_pop()?; + let low = state.stack_pop()?; + let insn_id = fun.push_insn(block, Insn::NewRange { low, high, flag, state: exit_id }); + state.stack_push(insn_id); + } + YARVINSN_invokebuiltin => { + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let mut args = vec![]; - for _ in 0..argc { + for _ in 0..bf.argc { args.push(state.stack_pop()?); } + args.push(self_param); args.reverse(); + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { + bf, + recv: self_param, + args, + state: exit_id, + leaf, + return_type, + }); + state.stack_push(insn_id); + } + YARVINSN_opt_invokebuiltin_delegate | + YARVINSN_opt_invokebuiltin_delegate_leave => { + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let index = get_arg(pc, 1).as_usize(); + let argc = bf.argc as usize; + + let mut args = vec![self_param]; + for &local in state.locals().skip(index).take(argc) { + args.push(local); + } + + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { + bf, + recv: self_param, + args, + state: exit_id, + leaf, + return_type, + }); + state.stack_push(insn_id); + } + YARVINSN_objtostring => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let argc = unsafe { vm_ci_argc((*cd).ci) }; + assert_eq!(0, argc, "objtostring should not have args"); + let recv = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let send = fun.push_insn(block, Insn::Send { self_val: recv, call_info: CallInfo { method_name }, cd, blockiseq, args, state: exit_id }); - state.stack_push(send); + let objtostring = fun.push_insn(block, Insn::ObjToString { val: recv, cd, state: exit_id }); + state.stack_push(objtostring) + } + YARVINSN_anytostring => { + let str = state.stack_pop()?; + let val = state.stack_pop()?; + + let anytostring = fun.push_insn(block, Insn::AnyToString { val, str, state: exit_id }); + state.stack_push(anytostring); + } + YARVINSN_getspecial => { + let key = get_arg(pc, 0).as_u64(); + let svar = get_arg(pc, 1).as_u64(); + + if svar == 0 { + // TODO: Handle non-backref + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownSpecialVariable(key) }); + // End the block + break; + } else if svar & 0x01 != 0 { + // Handle symbol backrefs like $&, $`, $', $+ + let shifted_svar: u8 = (svar >> 1).try_into().unwrap(); + let symbol_type = SpecialBackrefSymbol::try_from(shifted_svar).expect("invalid backref symbol"); + let result = fun.push_insn(block, Insn::GetSpecialSymbol { symbol_type, state: exit_id }); + state.stack_push(result); + } else { + // Handle number backrefs like $1, $2, $3 + let result = fun.push_insn(block, Insn::GetSpecialNumber { nth: svar, state: exit_id }); + state.stack_push(result); + } + } + YARVINSN_expandarray => { + let num = get_arg(pc, 0).as_u64(); + let flag = get_arg(pc, 1).as_u64(); + if flag != 0 { + // We don't (yet) handle 0x01 (rest args), 0x02 (post args), or 0x04 + // (reverse?) + // + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block + } + let val = state.stack_pop()?; + let array = fun.push_insn(block, Insn::GuardType { val, guard_type: types::ArrayExact, state: exit_id, }); + let length = fun.push_insn(block, Insn::ArrayLength { array }); + fun.push_insn(block, Insn::GuardBitEquals { val: length, expected: Const::CInt64(num as i64), reason: SideExitReason::ExpandArray, state: exit_id }); + for i in (0..num).rev() { + // We do not emit a length guard here because in-bounds is already + // ensured by the expandarray length check above. + let index = fun.push_insn(block, Insn::Const { val: Const::CInt64(i.try_into().unwrap()) }); + let element = fun.push_insn(block, Insn::ArrayAref { array, index }); + state.stack_push(element); + } + } + _ => { + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode) }); + break; // End the block } - _ => return Err(ParseError::UnknownOpcode(insn_name(opcode as usize))), } if insn_idx_to_block.contains_key(&insn_idx) { let target = insn_idx_to_block[&insn_idx]; - fun.push_insn(block, Insn::Jump(BranchEdge { target, args: state.as_args() })); - queue.push_back((state, target, insn_idx)); + fun.push_insn(block, Insn::Jump(BranchEdge { target, args: state.as_args(self_param) })); + queue.push_back((state, target, insn_idx, local_inval)); break; // End the block } } } + fun.set_param_types(); fun.infer_types(); match get_option!(dump_hir_init) { - Some(DumpHIR::WithoutSnapshot) => println!("HIR:\n{}", FunctionPrinter::without_snapshot(&fun)), - Some(DumpHIR::All) => println!("HIR:\n{}", FunctionPrinter::with_snapshot(&fun)), - Some(DumpHIR::Debug) => println!("HIR:\n{:#?}", &fun), + Some(DumpHIR::WithoutSnapshot) => println!("Initial HIR:\n{}", FunctionPrinter::without_snapshot(&fun)), + Some(DumpHIR::All) => println!("Initial HIR:\n{}", FunctionPrinter::with_snapshot(&fun)), + Some(DumpHIR::Debug) => println!("Initial HIR:\n{:#?}", &fun), None => {}, } fun.profiles = Some(profiles); + if let Err(err) = fun.validate() { + debug!("ZJIT: {err:?}: Initial HIR:\n{}", FunctionPrinter::without_snapshot(&fun)); + return Err(ParseError::Validation(err)); + } Ok(fun) } +/// Compile an entry_block for the interpreter +fn compile_entry_block(fun: &mut Function, jit_entry_insns: &[u32], insn_idx_to_block: &HashMap<u32, BlockId>) { + let entry_block = fun.entry_block; + let (self_param, entry_state) = compile_entry_state(fun); + let mut pc: Option<InsnId> = None; + let &all_opts_passed_insn_idx = jit_entry_insns.last().unwrap(); + + // Check-and-jump for each missing optional PC + for &jit_entry_insn in jit_entry_insns.iter() { + if jit_entry_insn == all_opts_passed_insn_idx { + continue; + } + let target_block = insn_idx_to_block.get(&jit_entry_insn) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + // Load PC once at the start of the block, shared among all cases + let pc = *pc.get_or_insert_with(|| fun.push_insn(entry_block, Insn::LoadPC)); + let expected_pc = fun.push_insn(entry_block, Insn::Const { + val: Const::CPtr(unsafe { rb_iseq_pc_at_idx(fun.iseq, jit_entry_insn) } as *const u8), + }); + let test_id = fun.push_insn(entry_block, Insn::IsBitEqual { left: pc, right: expected_pc }); + fun.push_insn(entry_block, Insn::IfTrue { + val: test_id, + target: BranchEdge { target: target_block, args: entry_state.as_args(self_param) }, + }); + } + + // Terminate the block with a jump to the block with all optionals passed + let target_block = insn_idx_to_block.get(&all_opts_passed_insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + fun.push_insn(entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile initial locals for an entry_block for the interpreter +fn compile_entry_state(fun: &mut Function) -> (InsnId, FrameState) { + let entry_block = fun.entry_block; + fun.push_insn(entry_block, Insn::EntryPoint { jit_entry_idx: None }); + + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + let self_param = fun.push_insn(entry_block, Insn::LoadSelf); + let mut entry_state = FrameState::new(iseq); + for local_idx in 0..num_locals(iseq) { + if local_idx < param_size { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx) as u32; + let use_sp = !iseq_escapes_ep(iseq); // If the ISEQ does not escape EP, we can assume EP + 1 == SP + let rest_param = Some(local_idx as i32) == rest_param_idx; + entry_state.locals.push(fun.push_insn(entry_block, Insn::GetLocal { level: 0, ep_offset, use_sp, rest_param })); + } else { + entry_state.locals.push(fun.push_insn(entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +/// Compile a jit_entry_block +fn compile_jit_entry_block(fun: &mut Function, jit_entry_idx: usize, target_block: BlockId) { + let jit_entry_block = fun.jit_entry_blocks[jit_entry_idx]; + fun.push_insn(jit_entry_block, Insn::EntryPoint { jit_entry_idx: Some(jit_entry_idx) }); + + // Prepare entry_state with basic block params + let (self_param, entry_state) = compile_jit_entry_state(fun, jit_entry_block, jit_entry_idx); + + // Jump to target_block + fun.push_insn(jit_entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile params and initial locals for a jit_entry_block +fn compile_jit_entry_state(fun: &mut Function, jit_entry_block: BlockId, jit_entry_idx: usize) -> (InsnId, FrameState) { + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let opt_num: usize = params.opt_num.try_into().expect("iseq param opt_num >= 0"); + let lead_num: usize = params.lead_num.try_into().expect("iseq param lead_num >= 0"); + let passed_opt_num = jit_entry_idx; + + // If the iseq has keyword parameters, the keyword bits local will be appended to the local table. + let kw_bits_idx: Option<usize> = if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if !keyword.is_null() { + Some(unsafe { (*keyword).bits_start } as usize) + } else { + None + } + } else { + None + }; + + let self_param = fun.push_insn(jit_entry_block, Insn::Param); + let mut entry_state = FrameState::new(iseq); + for local_idx in 0..num_locals(iseq) { + if (lead_num + passed_opt_num..lead_num + opt_num).contains(&local_idx) { + // Omitted optionals are locals, so they start as nils before their code run + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } else if Some(local_idx) == kw_bits_idx { + // We currently only support required keywords so the unspecified bits will always be zero. + // TODO: Make this a parameter when we start writing anything other than zero. + let unspecified_bits = VALUE::fixnum_from_usize(0); + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(unspecified_bits) })); + } else if local_idx < param_size { + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Param)); + } else { + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +pub struct Dominators<'a> { + f: &'a Function, + dominators: Vec<Vec<BlockId>>, +} + +impl<'a> Dominators<'a> { + pub fn new(f: &'a Function) -> Self { + let mut cfi = ControlFlowInfo::new(f); + Self::with_cfi(f, &mut cfi) + } + + pub fn with_cfi(f: &'a Function, cfi: &mut ControlFlowInfo) -> Self { + let block_ids = f.rpo(); + let mut dominators = vec![vec![]; f.blocks.len()]; + + // Compute dominators for each node using fixed point iteration. + // Approach can be found in Figure 1 of: + // https://www.cs.tufts.edu/~nr/cs257/archive/keith-cooper/dom14.pdf + // + // Initially we set: + // + // dom(entry) = {entry} for each entry block + // dom(b != entry) = {all nodes} + // + // Iteratively, apply: + // + // dom(b) = {b} union intersect(dom(p) for p in predecessors(b)) + // + // When we've run the algorithm and the dominator set no longer changes + // between iterations, then we have found the dominator sets. + + // Set up entry blocks. + // Entry blocks are only dominated by themselves. + for entry_block in &f.entry_blocks() { + dominators[entry_block.0] = vec![*entry_block]; + } + + // Setup the initial dominator sets. + for block_id in &block_ids { + if !f.entry_blocks().contains(block_id) { + // Non entry blocks are initially dominated by all other blocks. + dominators[block_id.0] = block_ids.clone(); + } + } + + let mut changed = true; + while changed { + changed = false; + + for block_id in &block_ids { + if *block_id == f.entry_block { + continue; + } + + // Get all predecessors for a given block. + let block_preds: Vec<BlockId> = cfi.predecessors(*block_id).collect(); + if block_preds.is_empty() { + continue; + } + + let mut new_doms = dominators[block_preds[0].0].clone(); + + // Compute the intersection of predecessor dominator sets into `new_doms`. + for pred_id in &block_preds[1..] { + let pred_doms = &dominators[pred_id.0]; + // Only keep a dominator in `new_doms` if it is also found in pred_doms + new_doms.retain(|d| pred_doms.contains(d)); + } + + // Insert sorted into `new_doms`. + match new_doms.binary_search(block_id) { + Ok(_) => {} + Err(pos) => new_doms.insert(pos, *block_id) + } + + // If we have computed a new dominator set, then we can update + // the dominators and mark that we need another iteration. + if dominators[block_id.0] != new_doms { + dominators[block_id.0] = new_doms; + changed = true; + } + } + } + + Self { f, dominators } + } + + + pub fn is_dominated_by(&self, left: BlockId, right: BlockId) -> bool { + self.dominators(left).any(|&b| b == right) + } + + pub fn dominators(&self, block: BlockId) -> Iter<'_, BlockId> { + self.dominators[block.0].iter() + } +} + +pub struct ControlFlowInfo<'a> { + function: &'a Function, + successor_map: HashMap<BlockId, Vec<BlockId>>, + predecessor_map: HashMap<BlockId, Vec<BlockId>>, +} + +impl<'a> ControlFlowInfo<'a> { + pub fn new(function: &'a Function) -> Self { + let mut successor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + let mut predecessor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + let uf = function.union_find.borrow(); + + for block_id in function.rpo() { + let block = &function.blocks[block_id.0]; + + // Since ZJIT uses extended basic blocks, one must check all instructions + // for their ability to jump to another basic block, rather than just + // the instructions at the end of a given basic block. + // + // Use BTreeSet to avoid duplicates and maintain an ordering. Also + // `BTreeSet<BlockId>` provides conversion trivially back to an `Vec<BlockId>`. + // Ordering is important so that the expect tests that serialize the predecessors + // and successors don't fail intermittently. + // todo(aidenfoxivey): Use `BlockSet` in lieu of `BTreeSet<BlockId>` + let successors: BTreeSet<BlockId> = block + .insns + .iter() + .map(|&insn_id| uf.find_const(insn_id)) + .filter_map(|insn_id| { + Self::extract_jump_target(&function.insns[insn_id.0]) + }) + .collect(); + + // Update predecessors for successor blocks. + for &succ_id in &successors { + predecessor_map + .entry(succ_id) + .or_default() + .push(block_id); + } + + // Store successors for this block. + // Convert successors from a `BTreeSet<BlockId>` to a `Vec<BlockId>`. + successor_map.insert(block_id, successors.iter().copied().collect()); + } + + Self { + function, + successor_map, + predecessor_map, + } + } + + pub fn is_succeeded_by(&self, left: BlockId, right: BlockId) -> bool { + self.successor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn is_preceded_by(&self, left: BlockId, right: BlockId) -> bool { + self.predecessor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn predecessors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.predecessor_map.get(&block).into_iter().flatten().copied() + } + + pub fn successors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.successor_map.get(&block).into_iter().flatten().copied() + } + + /// Helper function to extract the target of a jump instruction. + fn extract_jump_target(insn: &Insn) -> Option<BlockId> { + match insn { + Insn::Jump(target) + | Insn::IfTrue { target, .. } + | Insn::IfFalse { target, .. } => Some(target.target), + _ => None, + } + } +} + +pub struct LoopInfo<'a> { + cfi: &'a ControlFlowInfo<'a>, + dominators: &'a Dominators<'a>, + loop_depths: HashMap<BlockId, u32>, + loop_headers: BlockSet, + back_edge_sources: BlockSet, +} + +impl<'a> LoopInfo<'a> { + pub fn new(cfi: &'a ControlFlowInfo<'a>, dominators: &'a Dominators<'a>) -> Self { + let mut loop_headers: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let mut loop_depths: HashMap<BlockId, u32> = HashMap::new(); + let mut back_edge_sources: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let rpo = cfi.function.rpo(); + + for &block in &rpo { + loop_depths.insert(block, 0); + } + + // Collect loop headers. + for &block in &rpo { + // Initialize the loop depths. + for predecessor in cfi.predecessors(block) { + if dominators.is_dominated_by(predecessor, block) { + // Found a loop header, so then identify the natural loop. + loop_headers.insert(block); + back_edge_sources.insert(predecessor); + let loop_blocks = Self::find_natural_loop(cfi, block, predecessor); + // Increment the loop depth. + for loop_block in &loop_blocks { + *loop_depths.get_mut(loop_block).expect("Loop block should be populated.") += 1; + } + } + } + } + + Self { + cfi, + dominators, + loop_depths, + loop_headers, + back_edge_sources, + } + } + + fn find_natural_loop( + cfi: &ControlFlowInfo, + header: BlockId, + back_edge_source: BlockId, + ) -> HashSet<BlockId> { + // todo(aidenfoxivey): Reimplement using BlockSet + let mut loop_blocks = HashSet::new(); + let mut stack = vec![back_edge_source]; + + loop_blocks.insert(header); + loop_blocks.insert(back_edge_source); + + while let Some(block) = stack.pop() { + for pred in cfi.predecessors(block) { + // Pushes to stack only if `pred` wasn't already in `loop_blocks`. + if loop_blocks.insert(pred) { + stack.push(pred) + } + } + } + + loop_blocks + } + + pub fn loop_depth(&self, block: BlockId) -> u32 { + self.loop_depths.get(&block).copied().unwrap_or(0) + } + + pub fn is_back_edge_source(&self, block: BlockId) -> bool { + self.back_edge_sources.get(block) + } + + pub fn is_loop_header(&self, block: BlockId) -> bool { + self.loop_headers.get(block) + } +} + #[cfg(test)] mod union_find_tests { use super::UnionFind; @@ -2175,7 +7171,7 @@ mod rpo_tests { fn jump() { let mut function = Function::new(std::ptr::null()); let entry = function.entry_block; - let exit = function.new_block(); + let exit = function.new_block(0); function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); function.push_insn(entry, Insn::Return { val }); @@ -2186,8 +7182,8 @@ mod rpo_tests { fn diamond_iftrue() { let mut function = Function::new(std::ptr::null()); let entry = function.entry_block; - let side = function.new_block(); - let exit = function.new_block(); + let side = function.new_block(0); + let exit = function.new_block(0); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); function.push_insn(entry, Insn::IfTrue { val, target: BranchEdge { target: side, args: vec![] } }); @@ -2201,8 +7197,8 @@ mod rpo_tests { fn diamond_iffalse() { let mut function = Function::new(std::ptr::null()); let entry = function.entry_block; - let side = function.new_block(); - let exit = function.new_block(); + let side = function.new_block(0); + let exit = function.new_block(0); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); @@ -2222,6 +7218,165 @@ mod rpo_tests { } #[cfg(test)] +mod validation_tests { + use super::*; + + #[track_caller] + fn assert_matches_err(res: Result<(), ValidationError>, expected: ValidationError) { + match res { + Err(validation_err) => { + assert_eq!(validation_err, expected); + } + Ok(_) => panic!("Expected validation error"), + } + } + + #[test] + fn one_block_no_terminator() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + assert_matches_err(function.validate(), ValidationError::BlockHasNoTerminator(entry)); + } + + #[test] + fn one_block_terminator_not_at_end() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let insn_id = function.push_insn(entry, Insn::Return { val }); + function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + assert_matches_err(function.validate(), ValidationError::TerminatorNotAtEnd(entry, insn_id, 1)); + } + + #[test] + fn iftrue_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::IfTrue { val, target: BranchEdge { target: side, args: vec![val, val, val] } }); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn iffalse_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![val, val, val] } }); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn jump_mismatch_args() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Jump ( BranchEdge { target: side, args: vec![val, val, val] } )); + assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); + } + + #[test] + fn not_defined_within_bb() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + // Create an instruction without making it belong to anything. + let dangling = function.new_insn(Insn::Const{val: Const::CBool(true)}); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: dangling, state: InsnId(0usize) }); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, dangling)); + } + + #[test] + fn using_non_output_insn() { + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let const_ = function.push_insn(function.entry_block, Insn::Const{val: Const::CBool(true)}); + // Ret is a non-output instruction. + let ret = function.push_insn(function.entry_block, Insn::Return { val: const_ }); + let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: ret, state: InsnId(0usize) }); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, ret)); + } + + #[test] + fn not_dominated_by_diamond() { + // This tests that one branch is missing a definition which fails. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val1 = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::IfFalse { val: val1, target: BranchEdge { target: side, args: vec![] } }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val2 = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(exit, val2, v0)); + }); + } + + #[test] + fn dominated_by_diamond() { + // This tests that both branches with a definition succeeds. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let side = function.new_block(0); + let exit = function.new_block(0); + let v0 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); + function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); + function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); + function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + let _val = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + // Just checking that we don't panic. + assert!(function.validate_definite_assignment().is_ok()); + }); + } + + #[test] + fn instruction_appears_twice_in_same_block() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn_id(block, val); + function.push_insn(block, Insn::Return { val }); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val)); + } + + #[test] + fn instruction_appears_twice_with_different_ids() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val0 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let val1 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + function.make_equal_to(val1, val0); + function.push_insn(block, Insn::Return { val: val0 }); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val0)); + } + + #[test] + fn instruction_appears_twice_in_different_blocks() { + let mut function = Function::new(std::ptr::null()); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let exit = function.new_block(0); + function.push_insn(block, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + function.push_insn_id(exit, val); + function.push_insn(exit, Insn::Return { val }); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(exit, val)); + } +} + +#[cfg(test)] mod infer_tests { use super::*; @@ -2239,7 +7394,7 @@ mod infer_tests { fn test_const() { let mut function = Function::new(std::ptr::null()); let val = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qnil) }); - assert_bit_equal(function.infer_type(val), types::NilClassExact); + assert_bit_equal(function.infer_type(val), types::NilClass); } #[test] @@ -2276,17 +7431,6 @@ mod infer_tests { } #[test] - fn test_unknown() { - crate::cruby::with_rubyvm(|| { - let mut function = Function::new(std::ptr::null()); - let param = function.push_insn(function.entry_block, Insn::PutSelf); - let val = function.push_insn(function.entry_block, Insn::Test { val: param }); - function.infer_types(); - assert_bit_equal(function.type_of(val), types::CBool); - }); - } - - #[test] fn newarray() { let mut function = Function::new(std::ptr::null()); // Fake FrameState index of 0usize @@ -2307,15 +7451,15 @@ mod infer_tests { fn diamond_iffalse_merge_fixnum() { let mut function = Function::new(std::ptr::null()); let entry = function.entry_block; - let side = function.new_block(); - let exit = function.new_block(); + let side = function.new_block(0); + let exit = function.new_block(0); let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(4)) }); function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![v1] })); - let param = function.push_insn(exit, Insn::Param { idx: 0 }); + let param = function.push_insn(exit, Insn::Param); crate::cruby::with_rubyvm(|| { function.infer_types(); }); @@ -2326,1775 +7470,135 @@ mod infer_tests { fn diamond_iffalse_merge_bool() { let mut function = Function::new(std::ptr::null()); let entry = function.entry_block; - let side = function.new_block(); - let exit = function.new_block(); + let side = function.new_block(0); + let exit = function.new_block(0); let v0 = function.push_insn(side, Insn::Const { val: Const::Value(Qtrue) }); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(Qfalse) }); function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![v1] })); - let param = function.push_insn(exit, Insn::Param { idx: 0 }); + let param = function.push_insn(exit, Insn::Param); crate::cruby::with_rubyvm(|| { function.infer_types(); - assert_bit_equal(function.type_of(param), types::TrueClassExact.union(types::FalseClassExact)); + assert_bit_equal(function.type_of(param), types::TrueClass.union(types::FalseClass)); }); } } #[cfg(test)] -mod tests { +mod graphviz_tests { use super::*; - use expect_test::{expect, Expect}; - - #[macro_export] - macro_rules! assert_matches { - ( $x:expr, $pat:pat ) => { - { - let val = $x; - if (!matches!(val, $pat)) { - eprintln!("{} ({:?}) does not match pattern {}", stringify!($x), val, stringify!($pat)); - assert!(false); - } - } - }; - } - - - #[track_caller] - fn assert_matches_value(insn: Option<&Insn>, val: VALUE) { - match insn { - Some(Insn::Const { val: Const::Value(spec) }) => { - assert_eq!(*spec, val); - } - _ => assert!(false, "Expected Const {val}, found {insn:?}"), - } - } - - #[track_caller] - fn assert_matches_const(insn: Option<&Insn>, expected: Const) { - match insn { - Some(Insn::Const { val }) => { - assert_eq!(*val, expected, "{val:?} does not match {expected:?}"); - } - _ => assert!(false, "Expected Const {expected:?}, found {insn:?}"), - } - } - - #[track_caller] - fn assert_method_hir(method: &str, hir: Expect) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq(method)); - unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; - let function = iseq_to_hir(iseq).unwrap(); - assert_function_hir(function, hir); - } - - #[track_caller] - pub fn assert_function_hir(function: Function, expected_hir: Expect) { - let actual_hir = format!("{}", FunctionPrinter::without_snapshot(&function)); - expected_hir.assert_eq(&actual_hir); - } + use insta::assert_snapshot; #[track_caller] - fn assert_compile_fails(method: &str, reason: ParseError) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq(method)); - unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; - let result = iseq_to_hir(iseq); - assert!(result.is_err(), "Expected an error but succesfully compiled to HIR"); - assert_eq!(result.unwrap_err(), reason); - } - - - #[test] - fn test_putobject() { - eval("def test = 123"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:Fixnum[123] = Const Value(123) - Return v1 - "#]]); - } - - #[test] - fn test_new_array() { - eval("def test = []"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v2:ArrayExact = NewArray - Return v2 - "#]]); - } - - #[test] - fn test_new_array_with_element() { - eval("def test(a) = [a]"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v3:ArrayExact = NewArray v0 - Return v3 - "#]]); - } - - #[test] - fn test_new_array_with_elements() { - eval("def test(a, b) = [a, b]"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = NewArray v0, v1 - Return v4 - "#]]); - } - - #[test] - fn test_array_dup() { - eval("def test = [1, 2, 3]"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v3:ArrayExact = ArrayDup v1 - Return v3 - "#]]); - } - - // TODO(max): Test newhash when we have it - - #[test] - fn test_string_copy() { - eval("def test = \"hello\""); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v2:StringExact = StringCopy v1 - Return v2 - "#]]); - } - - #[test] - fn test_bignum() { - eval("def test = 999999999999999999999999999999999999"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:Bignum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - Return v1 - "#]]); - } - - #[test] - fn test_flonum() { - eval("def test = 1.5"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:Flonum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - Return v1 - "#]]); - } - - #[test] - fn test_heap_float() { - eval("def test = 1.7976931348623157e+308"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:HeapFloat[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - Return v1 - "#]]); - } - - #[test] - fn test_static_sym() { - eval("def test = :foo"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:StaticSymbol[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - Return v1 - "#]]); - } - - #[test] - fn test_opt_plus() { - eval("def test = 1+2"); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:Fixnum[1] = Const Value(1) - v2:Fixnum[2] = Const Value(2) - v4:BasicObject = SendWithoutBlock v1, :+, v2 - Return v4 - "#]]); - } - - #[test] - fn test_setlocal_getlocal() { - eval(" - def test - a = 1 - a - end - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v0:NilClassExact = Const Value(nil) - v2:Fixnum[1] = Const Value(1) - Return v2 - "#]]); - } - - #[test] - fn test_return_const() { - eval(" - def test(cond) - if cond - 3 - else - 4 - end - end - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v2:CBool = Test v0 - IfFalse v2, bb1(v0) - v4:Fixnum[3] = Const Value(3) - Return v4 - bb1(v6:BasicObject): - v8:Fixnum[4] = Const Value(4) - Return v8 - "#]]); - } - - #[test] - fn test_merge_const() { - eval(" - def test(cond) - if cond - result = 3 - else - result = 4 - end - result - end - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v1:NilClassExact = Const Value(nil) - v3:CBool = Test v0 - IfFalse v3, bb1(v0, v1) - v5:Fixnum[3] = Const Value(3) - Jump bb2(v0, v5) - bb1(v7:BasicObject, v8:NilClassExact): - v10:Fixnum[4] = Const Value(4) - Jump bb2(v7, v10) - bb2(v12:BasicObject, v13:Fixnum): - Return v13 - "#]]); - } - - #[test] - fn test_opt_plus_fixnum() { - eval(" - def test(a, b) = a + b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :+, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_minus_fixnum() { - eval(" - def test(a, b) = a - b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :-, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_mult_fixnum() { - eval(" - def test(a, b) = a * b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :*, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_div_fixnum() { - eval(" - def test(a, b) = a / b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :/, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_mod_fixnum() { - eval(" - def test(a, b) = a % b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :%, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_eq_fixnum() { - eval(" - def test(a, b) = a == b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :==, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_neq_fixnum() { - eval(" - def test(a, b) = a != b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :!=, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_lt_fixnum() { - eval(" - def test(a, b) = a < b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :<, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_le_fixnum() { - eval(" - def test(a, b) = a <= b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :<=, v1 - Return v4 - "#]]); - } - - #[test] - fn test_opt_gt_fixnum() { - eval(" - def test(a, b) = a > b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :>, v1 - Return v4 - "#]]); - } - - #[test] - fn test_loop() { - eval(" - def test - result = 0 - times = 10 - while times > 0 - result = result + 1 - times = times - 1 - end - result - end - test - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v0:NilClassExact = Const Value(nil) - v1:NilClassExact = Const Value(nil) - v3:Fixnum[0] = Const Value(0) - v4:Fixnum[10] = Const Value(10) - Jump bb2(v3, v4) - bb2(v6:BasicObject, v7:BasicObject): - v9:Fixnum[0] = Const Value(0) - v11:BasicObject = SendWithoutBlock v7, :>, v9 - v12:CBool = Test v11 - IfTrue v12, bb1(v6, v7) - v14:NilClassExact = Const Value(nil) - Return v6 - bb1(v16:BasicObject, v17:BasicObject): - v19:Fixnum[1] = Const Value(1) - v21:BasicObject = SendWithoutBlock v16, :+, v19 - v22:Fixnum[1] = Const Value(1) - v24:BasicObject = SendWithoutBlock v17, :-, v22 - Jump bb2(v21, v24) - "#]]); - } - - #[test] - fn test_opt_ge_fixnum() { - eval(" - def test(a, b) = a >= b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v0, :>=, v1 - Return v4 - "#]]); - } - - #[test] - fn test_display_types() { - eval(" - def test - cond = true - if cond - 3 - else - 4 - end - end - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v0:NilClassExact = Const Value(nil) - v2:TrueClassExact = Const Value(true) - v3:CBool[true] = Test v2 - IfFalse v3, bb1(v2) - v5:Fixnum[3] = Const Value(3) - Return v5 - bb1(v7): - v9 = Const Value(4) - Return v9 - "#]]); - } - - #[test] - fn test_send_without_block() { - eval(" - def bar(a, b) - a+b - end - def test - bar(2, 3) - end - test - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - v2:Fixnum[2] = Const Value(2) - v3:Fixnum[3] = Const Value(3) - v5:BasicObject = SendWithoutBlock v1, :bar, v2, v3 - Return v5 - "#]]); - } - - #[test] - fn test_send_with_block() { - eval(" - def test(a) - a.each {|item| - item - } - end - test([1,2,3]) - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v3:BasicObject = Send v0, 0x1000, :each - Return v3 - "#]]); - } - - #[test] - fn different_objects_get_addresses() { - eval("def test = unknown_method([0], [1], '2', '2')"); - - // The 2 string literals have the same address because they're deduped. - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - v2:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:ArrayExact = ArrayDup v2 - v5:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v7:ArrayExact = ArrayDup v5 - v8:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) - v9:StringExact = StringCopy v8 - v10:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) - v11:StringExact = StringCopy v10 - v13:BasicObject = SendWithoutBlock v1, :unknown_method, v4, v7, v9, v11 - Return v13 - "#]]); - } - - #[test] - fn test_cant_compile_splat() { - eval(" - def test(a) = foo(*a) - "); - assert_compile_fails("test", ParseError::UnknownOpcode("splatarray".into())) - } - - #[test] - fn test_cant_compile_block_arg() { - eval(" - def test(a) = foo(&a) - "); - assert_compile_fails("test", ParseError::UnhandledCallType(CallType::BlockArg)) - } - - #[test] - fn test_cant_compile_kwarg() { - eval(" - def test(a) = foo(a: 1) - "); - assert_compile_fails("test", ParseError::UnhandledCallType(CallType::Kwarg)) - } - - #[test] - fn test_cant_compile_kw_splat() { - eval(" - def test(a) = foo(**a) - "); - assert_compile_fails("test", ParseError::UnhandledCallType(CallType::KwSplat)) - } - - // TODO(max): Figure out how to generate a call with TAILCALL flag - - #[test] - fn test_cant_compile_super() { - eval(" - def test = super() - "); - assert_compile_fails("test", ParseError::UnknownOpcode("invokesuper".into())) - } - - #[test] - fn test_cant_compile_zsuper() { - eval(" - def test = super - "); - assert_compile_fails("test", ParseError::UnknownOpcode("invokesuper".into())) - } - - #[test] - fn test_cant_compile_super_forward() { - eval(" - def test(...) = super(...) - "); - assert_compile_fails("test", ParseError::UnknownOpcode("invokesuperforward".into())) - } - - // TODO(max): Figure out how to generate a call with OPT_SEND flag - - #[test] - fn test_cant_compile_kw_splat_mut() { - eval(" - def test(a) = foo **a, b: 1 - "); - assert_compile_fails("test", ParseError::UnknownOpcode("putspecialobject".into())) - } - - #[test] - fn test_cant_compile_splat_mut() { - eval(" - def test(*) = foo *, 1 - "); - assert_compile_fails("test", ParseError::UnknownOpcode("splatarray".into())) - } - - #[test] - fn test_cant_compile_forwarding() { - eval(" - def test(...) = foo(...) - "); - assert_compile_fails("test", ParseError::UnknownOpcode("sendforward".into())) - } - - #[test] - fn test_opt_new() { - eval(" - class C; end - def test = C.new - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = GetConstantPath 0x1000 - v2:NilClassExact = Const Value(nil) - Jump bb1(v2, v1) - bb1(v4:NilClassExact, v5:BasicObject): - v8:BasicObject = SendWithoutBlock v5, :new - Jump bb2(v8, v4) - bb2(v10:BasicObject, v11:NilClassExact): - Return v10 - "#]]); - } - - #[test] - fn test_opt_newarray_send_max_no_elements() { - eval(" - def test = [].max - "); - // TODO(max): Rewrite to nil - assert_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) - v3:BasicObject = ArrayMax - Return v3 - "#]]); - } - - #[test] - fn test_opt_newarray_send_max() { - eval(" - def test(a,b) = [a,b].max - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) - v5:BasicObject = ArrayMax v0, v1 - Return v5 - "#]]); - } - - #[test] - fn test_opt_length() { - eval(" - def test(a,b) = [a,b].length - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = NewArray v0, v1 - v6:BasicObject = SendWithoutBlock v4, :length - Return v6 - "#]]); - } - - #[test] - fn test_opt_size() { - eval(" - def test(a,b) = [a,b].size - "); - assert_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = NewArray v0, v1 - v6:BasicObject = SendWithoutBlock v4, :size - Return v6 - "#]]); - } -} - -#[cfg(test)] -mod opt_tests { - use super::*; - use super::tests::assert_function_hir; - use expect_test::{expect, Expect}; - - #[track_caller] - fn assert_optimized_method_hir(method: &str, hir: Expect) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq(method)); + fn hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; let mut function = iseq_to_hir(iseq).unwrap(); function.optimize(); - assert_function_hir(function, hir); - } - - #[test] - fn test_fold_iftrue_away() { - eval(" - def test - cond = true - if cond - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v5:Fixnum[3] = Const Value(3) - Return v5 - "#]]); - } - - #[test] - fn test_fold_iftrue_into_jump() { - eval(" - def test - cond = false - if cond - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v2:FalseClassExact = Const Value(false) - Jump bb1(v2) - bb1(v7:FalseClassExact): - v9:Fixnum[4] = Const Value(4) - Return v9 - "#]]); - } - - #[test] - fn test_fold_fixnum_add() { - eval(" - def test - 1 + 2 + 3 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v14:Fixnum[6] = Const Value(6) - Return v14 - "#]]); + function.validate().unwrap(); + format!("{}", FunctionGraphvizPrinter::new(&function)) } #[test] - fn test_fold_fixnum_less() { - eval(" - def test - if 1 < 2 - 3 - else - 4 - end - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v7:Fixnum[3] = Const Value(3) - Return v7 - "#]]); - } + fn test_guard_fixnum_or_fixnum() { + eval(r#" + def test(x, y) = x | y - #[test] - fn test_fold_fixnum_eq_true() { - eval(" - def test - if 1 == 2 - 3 - else - 4 - end - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - Jump bb1() - bb1(): - v10:Fixnum[4] = Const Value(4) - Return v10 - "#]]); + test(1, 2) + "#); + assert_snapshot!(hir_string("test"), @r#" + digraph G { # test@<compiled>:2 + node [shape=plaintext]; + mode=hier; overlap=false; splines=true; + bb0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb0() </TD></TR> + <TR><TD ALIGN="left" PORT="v0">EntryPoint interpreter </TD></TR> + <TR><TD ALIGN="left" PORT="v1">v1:BasicObject = LoadSelf </TD></TR> + <TR><TD ALIGN="left" PORT="v2">v2:BasicObject = GetLocal :x, l0, SP@5 </TD></TR> + <TR><TD ALIGN="left" PORT="v3">v3:BasicObject = GetLocal :y, l0, SP@4 </TD></TR> + <TR><TD ALIGN="left" PORT="v4">Jump bb2(v1, v2, v3) </TD></TR> + </TABLE>>]; + bb0:v4 -> bb2:params:n; + bb1 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v5">EntryPoint JIT(0) </TD></TR> + <TR><TD ALIGN="left" PORT="v9">Jump bb2(v6, v7, v8) </TD></TR> + </TABLE>>]; + bb1:v9 -> bb2:params:n; + bb2 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v15">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v18">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v24">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v25">PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) </TD></TR> + <TR><TD ALIGN="left" PORT="v26">v26:Fixnum = GuardType v11, Fixnum </TD></TR> + <TR><TD ALIGN="left" PORT="v27">v27:Fixnum = GuardType v12, Fixnum </TD></TR> + <TR><TD ALIGN="left" PORT="v28">v28:Fixnum = FixnumOr v26, v27 </TD></TR> + <TR><TD ALIGN="left" PORT="v29">IncrCounter inline_cfunc_optimized_send_count </TD></TR> + <TR><TD ALIGN="left" PORT="v21">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v22">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v23">Return v28 </TD></TR> + </TABLE>>]; + } + "#); } #[test] - fn test_fold_fixnum_eq_false() { - eval(" - def test - if 2 == 2 + fn test_multiple_blocks() { + eval(r#" + def test(c) + if c 3 else 4 end end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v7:Fixnum[3] = Const Value(3) - Return v7 - "#]]); - } - - #[test] - fn test_replace_guard_if_known_fixnum() { - eval(" - def test(a) - a + 1 - end - test(2); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v7:Fixnum = GuardType v0, Fixnum - v8:Fixnum = FixnumAdd v7, v2 - Return v8 - "#]]); - } - - #[test] - fn test_param_forms_get_bb_param() { - eval(" - def rest(*array) = array - def kw(k:) = k - def kw_rest(**k) = k - def post(*rest, post) = post - def block(&b) = nil - def forwardable(...) = nil - "); - - assert_optimized_method_hir("rest", expect![[r#" - fn rest: - bb0(v0:ArrayExact): - Return v0 - "#]]); - // extra hidden param for the set of specified keywords - assert_optimized_method_hir("kw", expect![[r#" - fn kw: - bb0(v0:BasicObject, v1:BasicObject): - Return v0 - "#]]); - assert_optimized_method_hir("kw_rest", expect![[r#" - fn kw_rest: - bb0(v0:BasicObject): - Return v0 - "#]]); - assert_optimized_method_hir("block", expect![[r#" - fn block: - bb0(v0:BasicObject): - v2:NilClassExact = Const Value(nil) - Return v2 - "#]]); - assert_optimized_method_hir("post", expect![[r#" - fn post: - bb0(v0:ArrayExact, v1:BasicObject): - Return v1 - "#]]); - assert_optimized_method_hir("forwardable", expect![[r#" - fn forwardable: - bb0(v0:BasicObject): - v2:NilClassExact = Const Value(nil) - Return v2 - "#]]); - } - - #[test] - fn test_optimize_top_level_call_into_send_direct() { - eval(" - def foo - end - def test - foo - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008) - v6:BasicObject[VALUE(0x1010)] = GuardBitEquals v1, VALUE(0x1010) - v7:BasicObject = SendWithoutBlockDirect v6, :foo (0x1018) - Return v7 - "#]]); - } - - #[test] - fn test_optimize_nonexistent_top_level_call() { - eval(" - def foo - end - def test - foo - end - test; test - undef :foo - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - v3:BasicObject = SendWithoutBlock v1, :foo - Return v3 - "#]]); - } - - #[test] - fn test_optimize_private_top_level_call() { - eval(" - def foo - end - private :foo - def test - foo - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008) - v6:BasicObject[VALUE(0x1010)] = GuardBitEquals v1, VALUE(0x1010) - v7:BasicObject = SendWithoutBlockDirect v6, :foo (0x1018) - Return v7 - "#]]); - } - - #[test] - fn test_optimize_top_level_call_with_overloaded_cme() { - eval(" - def test - Integer(3) - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - v2:Fixnum[3] = Const Value(3) - PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008) - v7:BasicObject[VALUE(0x1010)] = GuardBitEquals v1, VALUE(0x1010) - v8:BasicObject = SendWithoutBlockDirect v7, :Integer (0x1018), v2 - Return v8 - "#]]); - } - - #[test] - fn test_optimize_top_level_call_with_args_into_send_direct() { - eval(" - def foo a, b - end - def test - foo 1, 2 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008) - v8:BasicObject[VALUE(0x1010)] = GuardBitEquals v1, VALUE(0x1010) - v9:BasicObject = SendWithoutBlockDirect v8, :foo (0x1018), v2, v3 - Return v9 - "#]]); - } - - #[test] - fn test_optimize_top_level_sends_into_send_direct() { - eval(" - def foo - end - def bar - end - def test - foo - bar - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = PutSelf - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008) - v9:BasicObject[VALUE(0x1010)] = GuardBitEquals v1, VALUE(0x1010) - v10:BasicObject = SendWithoutBlockDirect v9, :foo (0x1018) - v4:BasicObject = PutSelf - PatchPoint MethodRedefined(Object@0x1000, bar@0x1020) - v12:BasicObject[VALUE(0x1010)] = GuardBitEquals v4, VALUE(0x1010) - v13:BasicObject = SendWithoutBlockDirect v12, :bar (0x1018) - Return v13 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_add_both_profiled() { - eval(" - def test(a, b) = a + b - test(1,2); test(3,4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v7:Fixnum = GuardType v0, Fixnum - v8:Fixnum = GuardType v1, Fixnum - v9:Fixnum = FixnumAdd v7, v8 - Return v9 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_add_left_profiled() { - eval(" - def test(a) = a + 1 - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v7:Fixnum = GuardType v0, Fixnum - v8:Fixnum = FixnumAdd v7, v2 - Return v8 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_add_right_profiled() { - eval(" - def test(a) = 1 + a - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v7:Fixnum = GuardType v0, Fixnum - v8:Fixnum = FixnumAdd v2, v7 - Return v8 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_lt_both_profiled() { - eval(" - def test(a, b) = a < b - test(1,2); test(3,4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v7:Fixnum = GuardType v0, Fixnum - v8:Fixnum = GuardType v1, Fixnum - v9:BoolExact = FixnumLt v7, v8 - Return v9 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_lt_left_profiled() { - eval(" - def test(a) = a < 1 - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v7:Fixnum = GuardType v0, Fixnum - v8:BoolExact = FixnumLt v7, v2 - Return v8 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_lt_right_profiled() { - eval(" - def test(a) = 1 < a - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v7:Fixnum = GuardType v0, Fixnum - v8:BoolExact = FixnumLt v2, v7 - Return v8 - "#]]); - } - - - #[test] - fn test_eliminate_new_array() { - eval(" - def test() - c = [] - 5 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v4:Fixnum[5] = Const Value(5) - Return v4 - "#]]); - } - - #[test] - fn test_eliminate_new_array_with_elements() { - eval(" - def test(a) - c = [a] - 5 - end - test(1); test(2) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_array_dup() { - eval(" - def test - c = [1, 2] - 5 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_putself() { - eval(" - def test() - c = self - 5 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v3:Fixnum[5] = Const Value(5) - Return v3 - "#]]); - } - - #[test] - fn test_eliminate_string_copy() { - eval(r#" - def test() - c = "abc" - 5 - end - test; test - "#); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v4:Fixnum[5] = Const Value(5) - Return v4 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_add() { - eval(" - def test(a, b) - a + b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_sub() { - eval(" - def test(a, b) - a - b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_mul() { - eval(" - def test(a, b) - a * b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_do_not_eliminate_fixnum_div() { - eval(" - def test(a, b) - a / b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_DIV) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v10:Fixnum = FixnumDiv v8, v9 - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_do_not_eliminate_fixnum_mod() { - eval(" - def test(a, b) - a % b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MOD) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v10:Fixnum = FixnumMod v8, v9 - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_lt() { - eval(" - def test(a, b) - a < b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_le() { - eval(" - def test(a, b) - a <= b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LE) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_gt() { - eval(" - def test(a, b) - a > b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GT) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_ge() { - eval(" - def test(a, b) - a >= b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GE) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - #[test] - fn test_eliminate_fixnum_eq() { - eval(" - def test(a, b) - a == b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v8:Fixnum = GuardType v0, Fixnum - v9:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_neq() { - eval(" - def test(a, b) - a != b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_NEQ) - v9:Fixnum = GuardType v0, Fixnum - v10:Fixnum = GuardType v1, Fixnum - v5:Fixnum[5] = Const Value(5) - Return v5 - "#]]); - } - - #[test] - fn test_do_not_eliminate_get_constant_path() { - eval(" - def test() - C - 5 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = GetConstantPath 0x1000 - v2:Fixnum[5] = Const Value(5) - Return v2 - "#]]); - } - - #[test] - fn kernel_itself_const() { - eval(" - def test(x) = x.itself - test(0) # profile test(1) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008) - v6:Fixnum = GuardType v0, Fixnum - v7:BasicObject = CCall itself@0x1010, v6 - Return v7 - "#]]); - } - - #[test] - fn kernel_itself_known_type() { - eval(" - def test = [].itself - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v2:ArrayExact = NewArray - PatchPoint MethodRedefined(Array@0x1000, itself@0x1008) - v7:BasicObject = CCall itself@0x1010, v2 - Return v7 - "#]]); - } - - #[test] - fn eliminate_kernel_itself() { - eval(" - def test - x = [].itself - 1 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint MethodRedefined(Array@0x1000, itself@0x1008) - v6:Fixnum[1] = Const Value(1) - Return v6 - "#]]); - } - - #[test] - fn eliminate_module_name() { - eval(" - module M; end - def test - x = M.name - 1 - end - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, M) - PatchPoint MethodRedefined(Module@0x1008, name@0x1010) - v5:Fixnum[1] = Const Value(1) - Return v5 - "#]]); - } - - #[test] - fn kernel_itself_argc_mismatch() { - eval(" - def test = 1.itself(0) - test rescue 0 - test rescue 0 - "); - // Not specialized - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:Fixnum[1] = Const Value(1) - v2:Fixnum[0] = Const Value(0) - v4:BasicObject = SendWithoutBlock v1, :itself, v2 - Return v4 - "#]]); - } - - #[test] - fn const_send_direct_integer() { - eval(" - def test(x) = 1.zero? - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1000, zero?@0x1008) - v7:BasicObject = SendWithoutBlockDirect v2, :zero? (0x1010) - Return v7 - "#]]); - } - - #[test] - fn class_known_send_direct_array() { - eval(" - def test(x) - a = [1,2,3] - a.first - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject): - v1:NilClassExact = Const Value(nil) - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v5:ArrayExact = ArrayDup v3 - PatchPoint MethodRedefined(Array@0x1008, first@0x1010) - v10:BasicObject = SendWithoutBlockDirect v5, :first (0x1018) - Return v10 - "#]]); - } - - #[test] - fn string_bytesize_simple() { - eval(" - def test = 'abc'.bytesize - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v2:StringExact = StringCopy v1 - PatchPoint MethodRedefined(String@0x1008, bytesize@0x1010) - v7:Fixnum = CCall bytesize@0x1018, v2 - Return v7 - "#]]); - } - - #[test] - fn dont_replace_get_constant_path_with_empty_ic() { - eval(" - def test = Kernel - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = GetConstantPath 0x1000 - Return v1 - "#]]); - } - - #[test] - fn dont_replace_get_constant_path_with_invalidated_ic() { - eval(" - def test = Kernel - test - Kernel = 5 - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - v1:BasicObject = GetConstantPath 0x1000 - Return v1 - "#]]); - } - - #[test] - fn replace_get_constant_path_with_const() { - eval(" - def test = Kernel - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, Kernel) - v5:BasicObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - Return v5 - "#]]); - } - - #[test] - fn replace_nested_get_constant_path_with_const() { - eval(" - module Foo - module Bar - class C - end - end - end - def test = Foo::Bar::C - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, Foo::Bar::C) - v5:BasicObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - Return v5 - "#]]); - } - - #[test] - fn test_opt_new_no_initialize() { - eval(" - class C; end - def test = C.new - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, C) - v16:BasicObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v2:NilClassExact = Const Value(nil) - Jump bb1(v2, v16) - bb1(v4:NilClassExact, v5:BasicObject[VALUE(0x1008)]): - v8:BasicObject = SendWithoutBlock v5, :new - Jump bb2(v8, v4) - bb2(v10:BasicObject, v11:NilClassExact): - Return v10 - "#]]); - } - - #[test] - fn test_opt_new_initialize() { - eval(" - class C - def initialize x - @x = x - end - end - def test = C.new 1 - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, C) - v18:BasicObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v2:NilClassExact = Const Value(nil) - v3:Fixnum[1] = Const Value(1) - Jump bb1(v2, v18, v3) - bb1(v5:NilClassExact, v6:BasicObject[VALUE(0x1008)], v7:Fixnum[1]): - v10:BasicObject = SendWithoutBlock v6, :new, v7 - Jump bb2(v10, v5) - bb2(v12:BasicObject, v13:NilClassExact): - Return v12 - "#]]); - } - - #[test] - fn test_opt_length() { - eval(" - def test(a,b) = [a,b].length - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = NewArray v0, v1 - v6:BasicObject = SendWithoutBlock v4, :length - Return v6 - "#]]); - } - - #[test] - fn test_opt_size() { - eval(" - def test(a,b) = [a,b].size - "); - assert_optimized_method_hir("test", expect![[r#" - fn test: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = NewArray v0, v1 - v6:BasicObject = SendWithoutBlock v4, :size - Return v6 - "#]]); + test("x") + "#); + assert_snapshot!(hir_string("test"), @r#" + digraph G { # test@<compiled>:3 + node [shape=plaintext]; + mode=hier; overlap=false; splines=true; + bb0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb0() </TD></TR> + <TR><TD ALIGN="left" PORT="v0">EntryPoint interpreter </TD></TR> + <TR><TD ALIGN="left" PORT="v1">v1:BasicObject = LoadSelf </TD></TR> + <TR><TD ALIGN="left" PORT="v2">v2:BasicObject = GetLocal :c, l0, SP@4 </TD></TR> + <TR><TD ALIGN="left" PORT="v3">Jump bb2(v1, v2) </TD></TR> + </TABLE>>]; + bb0:v3 -> bb2:params:n; + bb1 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb1(v5:BasicObject, v6:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v4">EntryPoint JIT(0) </TD></TR> + <TR><TD ALIGN="left" PORT="v7">Jump bb2(v5, v6) </TD></TR> + </TABLE>>]; + bb1:v7 -> bb2:params:n; + bb2 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb2(v8:BasicObject, v9:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v12">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v14">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v15">v15:CBool = Test v9 </TD></TR> + <TR><TD ALIGN="left" PORT="v16">IfFalse v15, bb3(v8, v9) </TD></TR> + <TR><TD ALIGN="left" PORT="v18">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v19">v19:Fixnum[3] = Const Value(3) </TD></TR> + <TR><TD ALIGN="left" PORT="v21">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v22">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v23">Return v19 </TD></TR> + </TABLE>>]; + bb2:v16 -> bb3:params:n; + bb3 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> + <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb3(v24:BasicObject, v25:BasicObject) </TD></TR> + <TR><TD ALIGN="left" PORT="v28">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v29">v29:Fixnum[4] = Const Value(4) </TD></TR> + <TR><TD ALIGN="left" PORT="v31">PatchPoint NoTracePoint </TD></TR> + <TR><TD ALIGN="left" PORT="v32">CheckInterrupts </TD></TR> + <TR><TD ALIGN="left" PORT="v33">Return v29 </TD></TR> + </TABLE>>]; + } + "#); } } diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs new file mode 100644 index 0000000000..60b4c25986 --- /dev/null +++ b/zjit/src/hir/opt_tests.rs @@ -0,0 +1,11123 @@ +#[cfg(test)] +mod hir_opt_tests { + use crate::hir::*; + + use crate::{hir_strings, options::*}; + use insta::assert_snapshot; + use crate::hir::tests::hir_build_tests::assert_contains_opcode; + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[test] + fn test_fold_iftrue_away() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v23:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_iftrue_into_jump() { + eval(" + def test + cond = false + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:FalseClass = Const Value(false) + CheckInterrupts + v33:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_add() { + eval(" + def test + 1 + 2 + 3 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v33:Fixnum[3] = Const Value(3) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v34:Fixnum[6] = Const Value(6) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_fixnum_sub() { + eval(" + def test + 5 - 3 - 1 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v33:Fixnum[2] = Const Value(2) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v34:Fixnum[1] = Const Value(1) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_fixnum_sub_large_negative_result() { + eval(" + def test + 0 - 1073741825 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[1073741825] = Const Value(1073741825) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v24:Fixnum[-1073741825] = Const Value(-1073741825) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mult() { + eval(" + def test + 6 * 7 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[6] = Const Value(6) + v12:Fixnum[7] = Const Value(7) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v24:Fixnum[42] = Const Value(42) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mult_zero() { + eval(" + def test(n) + 0 * n + n * 0 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :n, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v33:Fixnum = GuardType v9, Fixnum + v45:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count + v20:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v38:Fixnum = GuardType v9, Fixnum + v46:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count + PatchPoint MethodRedefined(Integer@0x1000, +@0x1038, cme:0x1040) + v47:Fixnum[0] = Const Value(0) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v47 + "); + } + + #[test] + fn test_fold_fixnum_less() { + eval(" + def test + if 1 < 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v40:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v22:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_fixnum_less_equal() { + eval(" + def test + if 1 <= 2 && 2 <= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v55:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v21:Fixnum[2] = Const Value(2) + v23:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v57:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v33:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_greater() { + eval(" + def test + if 2 > 1 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v40:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v22:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_fixnum_greater_equal() { + eval(" + def test + if 2 >= 1 && 2 >= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v55:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v21:Fixnum[2] = Const Value(2) + v23:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v57:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v33:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_eq_false() { + eval(" + def test + if 1 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v40:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v31:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_fold_fixnum_eq_true() { + eval(" + def test + if 2 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v40:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v22:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_fixnum_neq_true() { + eval(" + def test + if 1 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v41:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v22:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_fixnum_neq_false() { + eval(" + def test + if 2 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v41:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v31:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn neq_with_side_effect_not_elided () { + let result = eval(" + class CustomEq + attr_reader :count + + def ==(o) + @count = @count.to_i + 1 + self.equal?(o) + end + end + + def test(object) + # intentionally unused, but also can't assign to underscore + object != object + nil + end + + custom = CustomEq.new + test(custom) + test(custom) + + custom.count + "); + assert_eq!(VALUE::fixnum_from_usize(2), result); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:13: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :object, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(CustomEq@0x1000) + PatchPoint MethodRedefined(CustomEq@0x1000, !=@0x1008, cme:0x1010) + v28:HeapObject[class_exact:CustomEq] = GuardType v9, HeapObject[class_exact:CustomEq] + v29:BoolExact = CCallWithFrame v28, :BasicObject#!=@0x1038, v9 + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_replace_guard_if_known_fixnum() { + eval(" + def test(a) + a + 1 + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_param_forms_get_bb_param() { + eval(" + def rest(*array) = array + def kw(k:) = k + def kw_rest(**k) = k + def post(*rest, post) = post + def block(&b) = nil + "); + assert_snapshot!(hir_strings!("rest", "kw", "kw_rest", "block", "post"), @r" + fn rest@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :array, l0, SP@4, * + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:ArrayExact): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:ArrayExact): + CheckInterrupts + Return v9 + + fn kw@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :k, l0, SP@5 + v3:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:Fixnum[0] = Const Value(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + CheckInterrupts + Return v11 + + fn kw_rest@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :k, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + Return v9 + + fn block@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:NilClass = Const Value(nil) + CheckInterrupts + Return v13 + + fn post@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :rest, l0, SP@5, * + v3:BasicObject = GetLocal :post, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:ArrayExact, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:ArrayExact, v12:BasicObject): + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_optimize_top_level_call_into_send_direct() { + eval(" + def foo = [] + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_iseq() { + eval(" + def foo = 1 + alias bar foo + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_cfunc() { + eval(" + alias bar itself + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_inline_nonparam_local_return() { + // Methods that return non-parameter local variables should NOT be inlined, + // because the local variable index will be out of bounds for args. + // The method must have a parameter so param_size > 0, and return a local + // that's not a parameter so local_idx >= param_size. + // Use dead code (if false) to create a local without initialization instructions, + // resulting in just getlocal + leave which enters the inlining code path. + eval(" + def foo(a) + if false + x = nil + end + x + end + def test = foo(1) + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:8: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_to_aliased_cfunc() { + eval(" + class C < Array + alias fun_new_map map + end + def test(o) = o.fun_new_map {|e| e } + test C.new; test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, fun_new_map@0x1008, cme:0x1010) + v23:ArraySubclass[class_exact:C] = GuardType v9, ArraySubclass[class_exact:C] + v24:BasicObject = CCallWithFrame v23, :C#fun_new_map@0x1038, block=0x1040 + v15:BasicObject = GetLocal :o, l0, EP@3 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_nonexistent_top_level_call() { + eval(" + def foo + end + def test + foo + end + test; test + undef :foo + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: SendWithoutBlock: unsupported method type Null + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_private_top_level_call() { + eval(" + def foo = [] + private :foo + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_top_level_call_with_overloaded_cme() { + eval(" + def test + Integer(3) + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :Integer (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_top_level_call_with_args_into_send_direct() { + eval(" + def foo(a, b) = [] + def test + foo 1, 2 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_top_level_sends_into_send_direct() { + eval(" + def foo = [] + def bar = [] + def test + foo + bar + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v23:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v24:BasicObject = SendWithoutBlockDirect v23, :foo (0x1038) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, bar@0x1040, cme:0x1048) + v27:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v28:BasicObject = SendWithoutBlockDirect v27, :bar (0x1038) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_direct_no_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v19:BasicObject = SendWithoutBlockDirect v18, :foo (0x1038) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_direct_one_optional_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v21:BasicObject = SendWithoutBlockDirect v20, :foo (0x1038), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_direct_all_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3, 4 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v23:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_variadic_ccall() { + eval(" + def test + puts 'Hello' + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + PatchPoint NoSingletonClass(Object@0x1008) + PatchPoint MethodRedefined(Object@0x1008, puts@0x1010, cme:0x1018) + v22:HeapObject[class_exact*:Object@VALUE(0x1008)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1008)] + v23:BasicObject = CCallVariadic v22, :Kernel#puts@0x1040, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_optimize_fixnum_add_if_redefined() { + eval(" + class Integer + def +(other) + 100 + end + end + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + IncrCounter inline_iseq_optimized_send_count + v28:Fixnum[100] = Const Value(100) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_both_profiled() { + eval(" + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAdd v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_left_profiled() { + eval(" + def test(a) = a + 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_right_profiled() { + eval(" + def test(a) = 1 + a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumAdd v13, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn integer_aref_with_fixnum_emits_fixnum_aref() { + eval(" + def test(a, b) = a[b] + test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAref v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn elide_fixnum_aref() { + eval(" + def test + 1[2] + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_too_many_args() { + eval(" + def test = 1[2, 3] + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v14:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v23:BasicObject = CCallVariadic v10, :Integer#[]@0x1038, v12, v14 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_non_fixnum() { + eval(r#" + def test = 1["x"] + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint MethodRedefined(Integer@0x1008, []@0x1010, cme:0x1018) + v23:BasicObject = CCallVariadic v10, :Integer#[]@0x1040, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_both_profiled() { + eval(" + def test(a, b) = a < b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:BoolExact = FixnumLt v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_left_profiled() { + eval(" + def test(a) = a < 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BoolExact = FixnumLt v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_right_profiled() { + eval(" + def test(a) = 1 < a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BoolExact = FixnumLt v13, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_literals() { + eval(" + def test() + a = 2 + (1..a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v25:RangeExact = NewRangeFixnum v17 NewRangeInclusive v13 + CheckInterrupts + Return v25 + "); + } + + + #[test] + fn test_optimize_new_range_fixnum_exclusive_literals() { + eval(" + def test() + a = 2 + (1...a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v25:RangeExact = NewRangeFixnum v17 NewRangeExclusive v13 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_high_guarded() { + eval(" + def test(a) + (1..a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v13 NewRangeInclusive v21 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_high_guarded() { + eval(" + def test(a) + (1...a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v13 NewRangeExclusive v21 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_low_guarded() { + eval(" + def test(a) + (a..10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v21 NewRangeInclusive v14 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_low_guarded() { + eval(" + def test(a) + (a...10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v21:Fixnum = GuardType v9, Fixnum + v22:RangeExact = NewRangeFixnum v21 NewRangeExclusive v14 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_eliminate_new_array() { + eval(" + def test() + c = [] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_opt_aref_array() { + eval(" + arr = [1,2,3] + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []@0x1008, cme:0x1010) + v25:ArrayExact = GuardType v9, ArrayExact + v26:CInt64[0] = UnboxFixnum v14 + v27:CInt64 = ArrayLength v25 + v28:CInt64[0] = GuardLess v26, v27 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64[0] = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v25, v30 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + assert_snapshot!(inspect("test [1,2,3]"), @"1"); + } + + #[test] + fn test_opt_aref_hash() { + eval(" + arr = {0 => 4} + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []@0x1008, cme:0x1010) + v25:HashExact = GuardType v9, HashExact + v26:BasicObject = HashAref v25, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v26 + "); + assert_snapshot!(inspect("test({0 => 4})"), @"4"); + } + + #[test] + fn test_eliminate_new_range() { + eval(" + def test() + c = (1..2) + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:RangeExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_do_not_eliminate_new_range_non_fixnum() { + eval(" + def test() + _ = (-'a'..'b') + 0 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v14:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v16:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v17:StringExact = StringCopy v16 + v19:RangeExact = NewRange v14 NewRangeInclusive v17 + PatchPoint NoEPEscape(test) + v25:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_eliminate_new_array_with_elements() { + eval(" + def test(a) + c = [a] + 5 + end + test(1); test(2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v17:ArrayExact = NewArray v11 + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_eliminate_new_hash() { + eval(" + def test() + c = {} + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_eliminate_new_hash_with_elements() { + eval(" + def test(aval, bval) + c = {a: aval, b: bval} + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :aval, l0, SP@6 + v3:BasicObject = GetLocal :bval, l0, SP@5 + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:NilClass): + v19:StaticSymbol[:a] = Const Value(VALUE(0x1000)) + v22:StaticSymbol[:b] = Const Value(VALUE(0x1008)) + v25:HashExact = NewHash v19: v13, v22: v14 + PatchPoint NoEPEscape(test) + v31:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_array_dup() { + eval(" + def test + c = [1, 2] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_hash_dup() { + eval(" + def test + c = {a: 1, b: 2} + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_putself() { + eval(" + def test() + c = self + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_eliminate_string_copy() { + eval(r#" + def test() + c = "abc" + 5 + end + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact = StringCopy v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_fixnum_add() { + eval(" + def test(a, b) + a + b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_sub() { + eval(" + def test(a, b) + a - b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_mul() { + eval(" + def test(a, b) + a * b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_div() { + eval(" + def test(a, b) + a / b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + v32:Fixnum = FixnumDiv v30, v31 + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_mod() { + eval(" + def test(a, b) + a % b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + v32:Fixnum = FixnumMod v30, v31 + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_lt() { + eval(" + def test(a, b) + a < b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_le() { + eval(" + def test(a, b) + a <= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_gt() { + eval(" + def test(a, b) + a > b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_ge() { + eval(" + def test(a, b) + a >= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_eq() { + eval(" + def test(a, b) + a == b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + v31:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_eliminate_fixnum_neq() { + eval(" + def test(a, b) + a != b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + v30:Fixnum = GuardType v11, Fixnum + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v32:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_do_not_eliminate_get_constant_path() { + eval(" + def test() + C + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + v15:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn kernel_itself_const() { + eval(" + def test(x) = x.itself + test(0) # profile + test(1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) + v21:Fixnum = GuardType v9, Fixnum + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v21 + "); + } + + #[test] + fn kernel_itself_known_type() { + eval(" + def test = [].itself + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v10 + "); + } + + #[test] + fn eliminate_kernel_itself() { + eval(" + def test + x = [].itself + 1 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + PatchPoint NoEPEscape(test) + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn eliminate_module_name() { + eval(" + module M; end + def test + x = M.name + 1 + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v29:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, name@0x1018, cme:0x1020) + IncrCounter inline_cfunc_optimized_send_count + v34:StringExact|NilClass = CCall v29, :Module#name@0x1048 + PatchPoint NoEPEscape(test) + v22:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn eliminate_array_length() { + eval(" + def test + [].length + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn normal_class_type_inference() { + eval(" + class C; end + def test = C + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v18:Class[C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn core_classes_type_inference() { + eval(" + def test = [String, Class, Module, BasicObject] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v29:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1010, Class) + v32:Class[Class@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1020, Module) + v35:Class[Module@0x1028] = Const Value(VALUE(0x1028)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1030, BasicObject) + v38:Class[BasicObject@0x1038] = Const Value(VALUE(0x1038)) + v22:ArrayExact = NewArray v29, v32, v35, v38 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn module_instances_are_module_exact() { + eval(" + def test = [Enumerable, Kernel] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Enumerable) + v23:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1010, Kernel) + v26:ModuleExact[VALUE(0x1018)] = Const Value(VALUE(0x1018)) + v16:ArrayExact = NewArray v23, v26 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn module_subclasses_are_not_module_exact() { + eval(" + class ModuleSubclass < Module; end + MY_MODULE = ModuleSubclass.new + def test = MY_MODULE + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_MODULE) + v18:ModuleSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn eliminate_array_size() { + eval(" + def test + [].size + 5 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn kernel_itself_argc_mismatch() { + eval(" + def test = 1.itself(0) + test rescue 0 + test rescue 0 + "); + // Not specialized + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[0] = Const Value(0) + v14:BasicObject = SendWithoutBlock v10, :itself, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_kernel_block_given_p() { + eval(" + def test = block_given? + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v20:BoolExact = IsBlockGiven + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_kernel_block_given_p_in_block() { + eval(" + TEST = proc { block_given? } + TEST.call + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn block in <compiled>@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v20:BoolExact = IsBlockGiven + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_elide_kernel_block_given_p() { + eval(" + def test + block_given? + 5 + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v23:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_cfunc_optimized_send_count + v15:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn const_send_direct_integer() { + eval(" + def test(x) = 1.zero? + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, zero?@0x1008, cme:0x1010) + IncrCounter inline_iseq_optimized_send_count + v23:BasicObject = InvokeBuiltin leaf <inline_expr>, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn class_known_send_direct_array() { + eval(" + def test(x) + a = [1,2,3] + a.first + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v16:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:ArrayExact = ArrayDup v16 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, first@0x1010, cme:0x1018) + IncrCounter inline_iseq_optimized_send_count + v31:BasicObject = InvokeBuiltin leaf <inline_expr>, v17 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn send_direct_to_module() { + eval(" + module M; end + def test = M.class + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v20:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, class@0x1018, cme:0x1020) + IncrCounter inline_iseq_optimized_send_count + v26:Class[Module@0x1010] = Const Value(VALUE(0x1010)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_to_instance_method() { + eval(" + class C + def foo = [] + end + + def test(c) = c.foo + c = C.new + test c + test c + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :c, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v22:BasicObject = SendWithoutBlockDirect v21, :foo (0x1038) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_block() { + eval(" + def foo(&block) = 1 + def test = foo {|| } + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :foo # SendFallbackReason: Send: unsupported method type Iseq + CheckInterrupts + Return v11 + "); + } + + #[test] + fn reload_local_across_send() { + eval(" + def foo(&block) = 1 + def test + a = 1 + foo {|| } + a + end + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + SetLocal :a, l0, EP@3, v13 + v19:BasicObject = Send v8, 0x1000, :foo # SendFallbackReason: Send: unsupported method type Iseq + v20:BasicObject = GetLocal :a, l0, EP@3 + v24:BasicObject = GetLocal :a, l0, EP@3 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_rest() { + eval(" + def foo(*args) = 1 + def test = foo 1 + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_param_rest + v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dont_specialize_call_to_post_param_iseq() { + eval(" + def foo(opt=80, post) = post + def test = foo(10) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[10] = Const Value(10) + IncrCounter complex_arg_pass_param_post + v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn specialize_call_to_iseq_with_multiple_required_kw() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v24:BasicObject = SendWithoutBlockDirect v22, :foo (0x1038), v11, v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn specialize_call_to_iseq_with_required_kw_reorder() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v24:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v26:BasicObject = SendWithoutBlockDirect v24, :foo (0x1038), v13, v15, v11 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_to_iseq_with_positional_and_required_kw_reorder() { + eval(" + def foo(x, a:, b:) = [x, a, b] + def test = foo(0, b: 2, a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v24:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v26:BasicObject = SendWithoutBlockDirect v24, :foo (0x1038), v11, v15, v13 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn dont_specialize_call_with_positional_and_optional_kw() { + eval(" + def foo(x, a: 1) = [x, a] + def test = foo(0, a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + IncrCounter complex_arg_pass_param_kw_opt + v15:BasicObject = SendWithoutBlock v6, :foo, v11, v13 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v15 + "); + } + + #[test] + fn specialize_call_with_pos_optional_and_req_kw() { + eval(" + def foo(r, x = 2, a:, b:) = [x, a] + def test = [foo(1, a: 3, b: 4), foo(1, 2, b: 4, a: 3)] # with and without the optional, change kw order + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + v15:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v37:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v39:BasicObject = SendWithoutBlockDirect v37, :foo (0x1038), v11, v13, v15 + v20:Fixnum[1] = Const Value(1) + v22:Fixnum[2] = Const Value(2) + v24:Fixnum[4] = Const Value(4) + v26:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v42:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + v44:BasicObject = SendWithoutBlockDirect v42, :foo (0x1038), v20, v22, v26, v24 + v30:ArrayExact = NewArray v39, v44 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_send_call_to_iseq_with_optional_kw() { + eval(" + def foo(a: 1) = a + def test = foo(a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + IncrCounter complex_arg_pass_param_kw_opt + v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_kwrest() { + eval(" + def foo(**args) = 1 + def test = foo(a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_param_kwrest + v13:BasicObject = SendWithoutBlock v6, :foo, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_optional_param_kw() { + eval(" + def foo(int: 1) = int + 1 + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter complex_arg_pass_param_kw_opt + v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_call_kwsplat() { + eval(" + def foo(a:) = a + def test = foo(**{a: 1}) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:HashExact = HashDup v11 + IncrCounter complex_arg_pass_caller_kw_splat + v14:BasicObject = SendWithoutBlock v6, :foo, v12 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v14 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_param_kwrest() { + eval(" + def foo(**kwargs) = kwargs.keys + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter complex_arg_pass_param_kwrest + v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn dont_optimize_ccall_with_kwarg() { + eval(" + def test = sprintf('%s', a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + v14:Fixnum[1] = Const Value(1) + v16:BasicObject = SendWithoutBlock v6, :sprintf, v12, v14 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v16 + "); + } + + #[test] + fn dont_optimize_ccall_with_block_and_kwarg() { + eval(" + def test(s) + a = [] + s.each_line(chomp: true) { |l| a << l } + a + end + test %(a\nb\nc) + test %() + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v16:ArrayExact = NewArray + SetLocal :a, l0, EP@3, v16 + v22:TrueClass = Const Value(true) + IncrCounter complex_arg_pass_caller_kwarg + v24:BasicObject = Send v11, 0x1000, :each_line, v22 # SendFallbackReason: Complex argument passing + v25:BasicObject = GetLocal :s, l0, EP@4 + v26:BasicObject = GetLocal :a, l0, EP@3 + v30:BasicObject = GetLocal :a, l0, EP@3 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_empty_ic() { + eval(" + def test = Kernel + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_invalidated_ic() { + eval(" + def test = Kernel + test + Kernel = 5 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn replace_get_constant_path_with_const() { + eval(" + def test = Kernel + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v18:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn replace_nested_get_constant_path_with_const() { + eval(" + module Foo + module Bar + class C + end + end + end + def test = Foo::Bar::C + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:8: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo::Bar::C) + v18:Class[Foo::Bar::C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_opt_new_no_initialize() { + eval(" + class C; end + def test = C.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v43:Class[C@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v46:HeapObject[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v50:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_initialize() { + eval(" + class C + def initialize x + @x = x + end + end + def test = C.new 1 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v46:Class[C@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + v16:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v49:HeapObject[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v52:BasicObject = SendWithoutBlockDirect v49, :initialize (0x1068), v16 + CheckInterrupts + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_opt_new_object() { + eval(" + def test = Object.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Object) + v43:Class[Object@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Object@0x1008, new@0x1009, cme:0x1010) + v46:ObjectExact = ObjectAllocClass Object:VALUE(0x1008) + PatchPoint NoSingletonClass(Object@0x1008) + PatchPoint MethodRedefined(Object@0x1008, initialize@0x1038, cme:0x1040) + v50:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_basic_object() { + eval(" + def test = BasicObject.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, BasicObject) + v43:Class[BasicObject@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(BasicObject@0x1008, new@0x1009, cme:0x1010) + v46:BasicObjectExact = ObjectAllocClass BasicObject:VALUE(0x1008) + PatchPoint NoSingletonClass(BasicObject@0x1008) + PatchPoint MethodRedefined(BasicObject@0x1008, initialize@0x1038, cme:0x1040) + v50:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_hash() { + eval(" + def test = Hash.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Hash) + v43:Class[Hash@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Hash@0x1008, new@0x1009, cme:0x1010) + v46:HashExact = ObjectAllocClass Hash:VALUE(0x1008) + IncrCounter complex_arg_pass_param_block + IncrCounter complex_arg_pass_param_kw_opt + v20:BasicObject = SendWithoutBlock v46, :initialize # SendFallbackReason: Complex argument passing + CheckInterrupts + CheckInterrupts + Return v46 + "); + assert_snapshot!(inspect("test"), @"{}"); + } + + #[test] + fn test_opt_new_array() { + eval(" + def test = Array.new 1 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Array) + v46:Class[Array@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + v16:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Array@0x1008, new@0x1009, cme:0x1010) + PatchPoint NoSingletonClass(Class@0x1038) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v57:BasicObject = CCallVariadic v46, :Array.new@0x1040, v16 + CheckInterrupts + Return v57 + "); + } + + #[test] + fn test_opt_new_set() { + eval(" + def test = Set.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Set) + v43:Class[Set@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Set@0x1008, new@0x1009, cme:0x1010) + v18:HeapBasicObject = ObjectAlloc v43 + PatchPoint NoSingletonClass(Set@0x1008) + PatchPoint MethodRedefined(Set@0x1008, initialize@0x1038, cme:0x1040) + v49:SetExact = GuardType v18, SetExact + v50:BasicObject = CCallVariadic v49, :Set#initialize@0x1068 + CheckInterrupts + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_opt_new_string() { + eval(" + def test = String.new + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v43:Class[String@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + PatchPoint MethodRedefined(String@0x1008, new@0x1009, cme:0x1010) + PatchPoint NoSingletonClass(Class@0x1038) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v54:BasicObject = CCallVariadic v43, :String.new@0x1040 + CheckInterrupts + Return v54 + "); + } + + #[test] + fn test_opt_new_regexp() { + eval(" + def test = Regexp.new '' + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Regexp) + v47:Class[Regexp@0x1008] = Const Value(VALUE(0x1008)) + v13:NilClass = Const Value(nil) + v16:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v17:StringExact = StringCopy v16 + PatchPoint MethodRedefined(Regexp@0x1008, new@0x1018, cme:0x1020) + v50:RegexpExact = ObjectAllocClass Regexp:VALUE(0x1008) + PatchPoint NoSingletonClass(Regexp@0x1008) + PatchPoint MethodRedefined(Regexp@0x1008, initialize@0x1048, cme:0x1050) + v54:BasicObject = CCallVariadic v50, :Regexp#initialize@0x1078, v17 + CheckInterrupts + CheckInterrupts + Return v50 + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + v29:CInt64 = ArrayLength v18 + v30:Fixnum = BoxFixnum v29 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + v29:CInt64 = ArrayLength v18 + v30:Fixnum = BoxFixnum v29 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_getblockparamproxy() { + eval(" + def test(&block) = tap(&block) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :block, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + GuardBlockParamProxy l0 + v15:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + v17:BasicObject = Send v8, 0x1008, :tap, v15 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + IncrCounter getivar_fallback_not_monomorphic + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_not_monomorphic + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_true() { + eval(" + @foo = 4 + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v15:HeapBasicObject = GuardType v6, HeapBasicObject + v16:CShape = LoadField v15, :_shape_id@0x1000 + v17:CShape[0x1001] = GuardBitEquals v16, CShape(0x1001) + v18:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_false() { + eval(" + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v15:HeapBasicObject = GuardType v6, HeapBasicObject + v16:CShape = LoadField v15, :_shape_id@0x1000 + v17:CShape[0x1001] = GuardBitEquals v16, CShape(0x1001) + v18:NilClass = Const Value(nil) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_specialize_proc_call() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.call(1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, call@0x1008, cme:0x1010) + v23:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v24:BasicObject = InvokeProc v23, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_proc_aref() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p[2] + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, []@0x1008, cme:0x1010) + v24:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v25:BasicObject = InvokeProc v24, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_proc_yield() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.yield(3) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, yield@0x1008, cme:0x1010) + v23:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v24:BasicObject = InvokeProc v23, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_proc_eqq() { + eval(" + p = proc { |x| x > 0 } + def test(p) + p === 1 + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1000) + PatchPoint MethodRedefined(Proc@0x1000, ===@0x1008, cme:0x1010) + v23:HeapObject[class_exact:Proc] = GuardType v9, HeapObject[class_exact:Proc] + v24:BasicObject = InvokeProc v23, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_specialize_proc_call_splat() { + eval(" + p = proc { } + def test(p) + empty = [] + p.call(*empty) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + v16:ArrayExact = NewArray + v22:ArrayExact = ToArray v16 + IncrCounter complex_arg_pass_caller_splat + v24:BasicObject = SendWithoutBlock v11, :call, v22 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_specialize_proc_call_kwarg() { + eval(" + p = proc { |a:| a } + def test(p) + p.call(a: 1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :p, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_caller_kwarg + v16:BasicObject = SendWithoutBlock v9, :call, v14 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_dont_specialize_definedivar_with_t_data() { + eval(" + class C < Range + def test = defined?(@a) + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter definedivar_fallback_not_t_object + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_definedivar() { + set_call_threshold(3); + eval(" + class C + def test = defined?(@a) + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter definedivar_fallback_not_monomorphic + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_definedivar() { + eval(r#" + class C + def test = defined?(@a) + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter definedivar_fallback_too_complex + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_already_in_shape() { + eval(" + @foo = 4 + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v19:HeapBasicObject = GuardType v6, HeapBasicObject + v20:CShape = LoadField v19, :_shape_id@0x1000 + v21:CShape[0x1001] = GuardBitEquals v20, CShape(0x1001) + StoreField v19, :@foo@0x1002, v10 + WriteBarrier v19, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_with_shape_transition() { + eval(" + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v19:HeapBasicObject = GuardType v6, HeapBasicObject + v20:CShape = LoadField v19, :_shape_id@0x1000 + v21:CShape[0x1001] = GuardBitEquals v20, CShape(0x1001) + StoreField v19, :@foo@0x1002, v10 + WriteBarrier v19, v10 + v24:CShape[0x1003] = Const CShape(0x1003) + StoreField v19, :_shape_id@0x1000, v24 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_multiple_monomorphic_setivar_with_shape_transition() { + eval(" + def test + @foo = 1 + @bar = 2 + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v25:HeapBasicObject = GuardType v6, HeapBasicObject + v26:CShape = LoadField v25, :_shape_id@0x1000 + v27:CShape[0x1001] = GuardBitEquals v26, CShape(0x1001) + StoreField v25, :@foo@0x1002, v10 + WriteBarrier v25, v10 + v30:CShape[0x1003] = Const CShape(0x1003) + StoreField v25, :_shape_id@0x1000, v30 + v16:Fixnum[2] = Const Value(2) + PatchPoint SingleRactorMode + v32:HeapBasicObject = GuardType v6, HeapBasicObject + v33:CShape = LoadField v32, :_shape_id@0x1000 + v34:CShape[0x1003] = GuardBitEquals v33, CShape(0x1003) + StoreField v32, :@bar@0x1004, v16 + WriteBarrier v32, v16 + v37:CShape[0x1005] = Const CShape(0x1005) + StoreField v32, :_shape_id@0x1000, v37 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_dont_specialize_setivar_with_t_data() { + eval(" + class C < Range + def test = @a = 5 + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_not_t_object + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_setivar() { + set_call_threshold(3); + eval(" + class C + def test = @a = 5 + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_not_monomorphic + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_setivar() { + eval(r#" + class C + def test = @a = 5 + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_too_complex + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_setivar_when_next_shape_is_too_complex() { + eval(r#" + class AboutToBeTooComplex + def test = @abc = 5 + end + SHAPE_MAX_VARIATIONS = 8 # see shape.h + SHAPE_MAX_VARIATIONS.times do + AboutToBeTooComplex.new.instance_variable_set(:"@a#{_1}", 1) + end + AboutToBeTooComplex.new.test + TEST = AboutToBeTooComplex.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + IncrCounter setivar_fallback_new_shape_too_complex + SetIvar v6, :@abc, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_hash() { + eval(" + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_dont_optimize_hash_freeze_if_redefined() { + eval(" + class Hash + def freeze; end + end + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_hash() { + eval(" + def test = {}.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_hash() { + eval(" + def test = {}.dup.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact = NewHash + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, dup@0x1008, cme:0x1010) + v22:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = SendWithoutBlock v22, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_hash_with_args() { + eval(" + def test = {}.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact = NewHash + v12:NilClass = Const Value(nil) + v14:BasicObject = SendWithoutBlock v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_ary() { + eval(" + def test = [].freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_ary() { + eval(" + def test = [].freeze.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_ary() { + eval(" + def test = [].dup.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, dup@0x1008, cme:0x1010) + v22:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = SendWithoutBlock v22, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_ary_with_args() { + eval(" + def test = [].freeze(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + v12:NilClass = Const Value(nil) + v14:BasicObject = SendWithoutBlock v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_str() { + eval(" + def test = ''.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_str() { + eval(" + def test = ''.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_str() { + eval(" + def test = ''.dup.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v23:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = SendWithoutBlock v23, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_no_elide_freeze_str_with_args() { + eval(" + def test = ''.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:NilClass = Const Value(nil) + v15:BasicObject = SendWithoutBlock v11, :freeze, v13 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_elide_uminus_with_frozen_str() { + eval(" + def test = -'' + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_uminus_with_refrozen_str() { + eval(" + def test = -''.freeze + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_uminus_with_unfrozen_str() { + eval(" + def test = -''.dup + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v23:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = SendWithoutBlock v23, :-@ # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_objtostring_anytostring_string() { + eval(r##" + def test = "#{('foo')}" + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v14:StringExact = StringCopy v13 + v21:StringExact = StringConcat v10, v14 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_objtostring_anytostring_with_non_string() { + eval(r##" + def test = "#{1}" + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled() { + eval(" + def test(a) + \"#{a}\" + end + test('foo'); test('foo') + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + v27:String = GuardType v9, String + v21:StringExact = StringConcat v13, v27 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled_string_subclass() { + eval(" + class MyString < String; end + + def test(a) + \"#{a}\" + end + foo = MyString.new('foo') + test(MyString.new(foo)); test(MyString.new(foo)) + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(MyString@0x1008) + v27:String = GuardType v9, String + v21:StringExact = StringConcat v13, v27 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_objtostring_profiled_nonstring_falls_back_to_send() { + eval(" + def test(a) + \"#{a}\" + end + test([1,2,3]); test([1,2,3]) # No fast path for array + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:ArrayExact = GuardType v9, ArrayExact + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, to_s@0x1010, cme:0x1018) + v31:BasicObject = CCallWithFrame v26, :Array#to_s@0x1040 + v19:String = AnyToString v9, str: v31 + v21:StringExact = StringConcat v13, v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_branchnil_nil() { + eval(" + def test + x = nil + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + CheckInterrupts + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_branchnil_truthy() { + eval(" + def test + x = 1 + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_dont_eliminate_load_from_non_frozen_array() { + eval(r##" + S = [4,5,6] + def test = S[0] + test + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v23:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, []@0x1018, cme:0x1020) + v27:CInt64[0] = UnboxFixnum v13 + v28:CInt64 = ArrayLength v23 + v29:CInt64[0] = GuardLess v27, v28 + v30:CInt64[0] = Const CInt64(0) + v31:CInt64[0] = GuardGreaterEq v29, v30 + v32:BasicObject = ArrayAref v23, v31 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v32 + "); + // TODO(max): Check the result of `S[0] = 5; test` using `inspect` to make sure that we + // actually do the load at run-time. + } + + #[test] + fn test_eliminate_load_from_frozen_array_in_bounds() { + eval(r##" + def test = [4,5,6].freeze[1] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[1] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[1] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[1] = GuardGreaterEq v26, v27 + v31:Fixnum[5] = Const Value(5) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative() { + eval(r##" + def test = [4,5,6].freeze[-3] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-3] = Const Value(-3) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[-3] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[-3] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[-3] = GuardGreaterEq v26, v27 + v31:Fixnum[4] = Const Value(4) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[-10] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-10] = Const Value(-10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[-10] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[-10] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[-10] = GuardGreaterEq v26, v27 + v31:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:CInt64[10] = UnboxFixnum v13 + v25:CInt64 = ArrayLength v11 + v26:CInt64[10] = GuardLess v24, v25 + v27:CInt64[0] = Const CInt64(0) + v28:CInt64[10] = GuardGreaterEq v26, v27 + v31:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_dont_optimize_array_aref_if_redefined() { + eval(r##" + class Array + def [](index) = [] + end + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v23:BasicObject = SendWithoutBlockDirect v11, :[] (0x1040), v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_optimize_array_aset_if_redefined() { + eval(r##" + class Array + def []=(*args); :redefined; end + end + + def test(arr) + arr[1] = 10 + end + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[1] = Const Value(1) + v18:Fixnum[10] = Const Value(10) + v22:BasicObject = SendWithoutBlock v9, :[]=, v16, v18 # SendFallbackReason: Uncategorized(opt_aset) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_dont_optimize_array_max_if_redefined() { + eval(r##" + class Array + def max = [] + end + def test = [4,5,6].max + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018) + v20:BasicObject = SendWithoutBlockDirect v11, :max (0x1040) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_set_type_from_constant() { + eval(" + MY_SET = Set.new + + def test = MY_SET + + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_SET) + v18:SetExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_regexp_type() { + eval(" + def test = /a/ + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:RegexpExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_bmethod_send_direct() { + eval(" + define_method(:zero) { :b } + define_method(:one) { |arg| arg } + + def test = one(zero) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, zero@0x1008, cme:0x1010) + v22:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v30:StaticSymbol[:b] = Const Value(VALUE(0x1038)) + PatchPoint SingleRactorMode + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, one@0x1040, cme:0x1048) + v27:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_symbol_block_bmethod() { + eval(" + define_method(:identity, &:itself) + def test = identity(100) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[100] = Const Value(100) + v13:BasicObject = SendWithoutBlock v6, :identity, v11 # SendFallbackReason: Bmethod: Proc object is not defined by an ISEQ + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_call_bmethod_with_block() { + eval(" + define_method(:bmethod) { :b } + def test = (bmethod {}) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :bmethod # SendFallbackReason: Send: unsupported method type Bmethod + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_call_shareable_bmethod() { + eval(" + class Foo + class << self + define_method(:identity, &(Ractor.make_shareable ->(val){val})) + end + end + def test = Foo.identity(100) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo) + v22:Class[Foo@0x1008] = Const Value(VALUE(0x1008)) + v13:Fixnum[100] = Const Value(100) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, identity@0x1018, cme:0x1020) + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_nil_nil_specialized_to_ccall() { + eval(" + def test = nil.nil? + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v20:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_eliminate_nil_nil_specialized_to_ccall() { + eval(" + def test + nil.nil? + 1 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_non_nil_nil_specialized_to_ccall() { + eval(" + def test = 1.nil? + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v20:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_eliminate_non_nil_nil_specialized_to_ccall() { + eval(" + def test + 1.nil? + 2 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + IncrCounter inline_cfunc_optimized_send_count + v17:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_guard_nil_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v22:NilClass = GuardType v9, NilClass + v23:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_false_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(false) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1000, nil?@0x1008, cme:0x1010) + v22:FalseClass = GuardType v9, FalseClass + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_true_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(true) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1000, nil?@0x1008, cme:0x1010) + v22:TrueClass = GuardType v9, TrueClass + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_symbol_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(:foo) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1000, nil?@0x1008, cme:0x1010) + v22:StaticSymbol = GuardType v9, StaticSymbol + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_fixnum_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v22:Fixnum = GuardType v9, Fixnum + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_float_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1.0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Float@0x1000, nil?@0x1008, cme:0x1010) + v22:Flonum = GuardType v9, Flonum + v23:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_guard_string_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test('foo') + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, nil?@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + v24:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_basicobject_not_truthy() { + eval(" + def test(a) = !a + + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, !@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_specialize_basicobject_not_false() { + eval(" + def test(a) = !a + + test(false) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1000, !@0x1008, cme:0x1010) + v22:FalseClass = GuardType v9, FalseClass + v23:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_basicobject_not_nil() { + eval(" + def test(a) = !a + + test(nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1000, !@0x1008, cme:0x1010) + v22:NilClass = GuardType v9, NilClass + v23:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_basicobject_not_falsy() { + eval(" + def test(a) = !(if a then false else nil end) + + # TODO(max): Make this not GuardType NilClass and instead just reason + # statically + test(false) + test(true) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + IfFalse v15, bb3(v8, v9) + v18:FalseClass = Const Value(false) + CheckInterrupts + Jump bb4(v8, v9, v18) + bb3(v22:BasicObject, v23:BasicObject): + v26:NilClass = Const Value(nil) + Jump bb4(v22, v23, v26) + bb4(v28:BasicObject, v29:BasicObject, v30:NilClass|FalseClass): + PatchPoint MethodRedefined(NilClass@0x1000, !@0x1008, cme:0x1010) + v41:NilClass = GuardType v30, NilClass + v42:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v42 + "); + } + + #[test] + fn test_specialize_array_empty_p() { + eval(" + def test(a) = a.empty? + + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, empty?@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:CInt64 = ArrayLength v23 + v25:CInt64[0] = Const CInt64(0) + v26:CBool = IsBitEqual v24, v25 + v27:BoolExact = BoxBool v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_specialize_hash_empty_p_to_ccall() { + eval(" + def test(a) = a.empty? + + test({}) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, empty?@0x1008, cme:0x1010) + v23:HashExact = GuardType v9, HashExact + IncrCounter inline_cfunc_optimized_send_count + v25:BoolExact = CCall v23, :Hash#empty?@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_basic_object_eq_to_ccall() { + eval(" + class C; end + def test(a, b) = a == b + + test(C.new, C.new) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, ==@0x1008, cme:0x1010) + v27:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C] + v28:CBool = IsBitEqual v27, v12 + v29:BoolExact = BoxBool v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_guard_fixnum_and_fixnum() { + eval(" + def test(x, y) = x & y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, &@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumAnd v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_guard_fixnum_or_fixnum() { + eval(" + def test(x, y) = x | y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:Fixnum = GuardType v12, Fixnum + v28:Fixnum = FixnumOr v26, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_method_redefinition_patch_point_on_top_level_method() { + eval(" + def foo; end + def test = foo + + test; test + "); + + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:NilClass = Const Value(nil) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_getivar_embedded() { + eval(" + class C + attr_reader :foo + def initialize + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:BasicObject = LoadField v21, :@foo@0x103a + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_getivar_extended() { + eval(r#" + class C + attr_reader :foo + def initialize + 1000.times do |i| + instance_variable_set("@v#{i}", i) + end + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:13: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:CPtr = LoadField v21, :_as_heap@0x103a + v27:BasicObject = LoadField v26, :@foo@0x103b + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_getivar_on_module() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :_shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) + v19:CUInt16[0] = Const CUInt16(0) + v20:BasicObject = CCall v16, :rb_ivar_get_at_no_ractor_check@0x1008, v19 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_on_class() { + eval(" + class C + @foo = 42 + def self.test = @foo + end + C.test + "); + assert_snapshot!(hir_string_proc("C.method(:test)"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :_shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) + v19:CUInt16[0] = Const CUInt16(0) + v20:BasicObject = CCall v16, :rb_ivar_get_at_no_ractor_check@0x1008, v19 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_on_t_data() { + eval(" + class C < Range + def test = @a + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :_shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) + v19:CUInt16[0] = Const CUInt16(0) + v20:BasicObject = CCall v16, :rb_ivar_get_at_no_ractor_check@0x1008, v19 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit UnhandledYARVInsn(getinstancevariable) + "); + } + + #[test] + fn test_optimize_attr_reader_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + class << self + attr_reader :foo + end + def self.test = foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = SendWithoutBlock v6, :foo # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_dont_optimize_getivar_polymorphic() { + set_call_threshold(3); + eval(" + class C + attr_reader :foo, :bar + + def foo_then_bar + @foo = 1 + @bar = 2 + end + + def bar_then_foo + @bar = 3 + @foo = 4 + end + end + + O1 = C.new + O1.foo_then_bar + O2 = C.new + O2.bar_then_foo + def test(o) = o.foo + test O1 + test O2 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:20: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:BasicObject = SendWithoutBlock v9, :foo # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_dont_optimize_getivar_with_too_complex_shape() { + eval(r#" + class C + attr_accessor :foo + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + def test(o) = o.foo + test obj + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:12: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + IncrCounter getivar_fallback_too_complex + v22:BasicObject = GetIvar v21, :@foo + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_send_with_block() { + eval(r#" + def test = [1, 2, 3].map { |x| x * 2 } + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, map@0x1010, cme:0x1018) + v21:BasicObject = CCallWithFrame v11, :Array#map@0x1040, block=0x1048 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_variadic_with_block() { + eval(r#" + A = [1, 2, 3] + B = ["a", "b", "c"] + + def test + result = [] + A.zip(B) { |x, y| result << [x, y] } + result + end + + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + SetLocal :result, l0, EP@3, v13 + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v36:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1010, B) + v39:ArrayExact[VALUE(0x1018)] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(Array@0x1020) + PatchPoint MethodRedefined(Array@0x1020, zip@0x1028, cme:0x1030) + v43:BasicObject = CCallVariadic v36, :zip@0x1058, v39 + v25:BasicObject = GetLocal :result, l0, EP@3 + v29:BasicObject = GetLocal :result, l0, EP@3 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_do_not_optimize_send_with_block_forwarding() { + eval(r#" + def test(&block) = [].map(&block) + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :block, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:ArrayExact = NewArray + GuardBlockParamProxy l0 + v16:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + IncrCounter complex_arg_pass_caller_blockarg + v18:BasicObject = Send v13, 0x1008, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_do_not_optimize_send_to_iseq_method_with_block() { + eval(r#" + def foo + yield 1 + end + + def test = foo {} + test; test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :foo # SendFallbackReason: Send: unsupported method type Iseq + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_attr_reader_constant() { + eval(" + class C + attr_reader :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v25:CShape = LoadField v20, :_shape_id@0x1048 + v26:CShape[0x1049] = GuardBitEquals v25, CShape(0x1049) + v27:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_attr_accessor_constant() { + eval(" + class C + attr_accessor :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v25:CShape = LoadField v20, :_shape_id@0x1048 + v26:CShape[0x1049] = GuardBitEquals v25, CShape(0x1049) + v27:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_attr_reader() { + eval(" + class C + attr_reader :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:NilClass = Const Value(nil) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_inline_attr_accessor() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:NilClass = Const Value(nil) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_inline_attr_accessor_set() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v29:CShape = LoadField v26, :_shape_id@0x1038 + v30:CShape[0x1039] = GuardBitEquals v29, CShape(0x1039) + StoreField v26, :@foo@0x103a, v16 + WriteBarrier v26, v16 + v33:CShape[0x103b] = Const CShape(0x103b) + StoreField v26, :_shape_id@0x1038, v33 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_inline_attr_writer_set() { + eval(" + class C + attr_writer :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v29:CShape = LoadField v26, :_shape_id@0x1038 + v30:CShape[0x1039] = GuardBitEquals v29, CShape(0x1039) + StoreField v26, :@foo@0x103a, v16 + WriteBarrier v26, v16 + v33:CShape[0x103b] = Const CShape(0x103b) + StoreField v26, :_shape_id@0x1038, v33 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_inline_struct_aref_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v22:BasicObject = LoadField v21, :foo@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_inline_struct_aref_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v22:CPtr = LoadField v21, :_as_heap@0x1038 + v23:BasicObject = LoadField v22, :foo@0x1039 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_elide_struct_aref() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) + o.foo + 5 + end + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) + v25:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_inline_struct_aset_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@5 + v3:BasicObject = GetLocal :v, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v29:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C] + v30:HeapObject[class_exact:C] = GuardNotFrozen v29 + StoreField v30, :foo=@0x1038, v12 + WriteBarrier v30, v12 + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_inline_struct_aset_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@5 + v3:BasicObject = GetLocal :v, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, foo=@0x1008, cme:0x1010) + v29:HeapObject[class_exact:C] = GuardType v11, HeapObject[class_exact:C] + v30:HeapObject[class_exact:C] = GuardNotFrozen v29 + v31:CPtr = LoadField v30, :_as_heap@0x1038 + StoreField v31, :foo=@0x1039, v12 + WriteBarrier v30, v12 + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_array_reverse_returns_array() { + eval(r#" + def test = [].reverse + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v20:ArrayExact = CCallWithFrame v10, :Array#reverse@0x1038 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_array_reverse_is_elidable() { + eval(r#" + def test + [].reverse + 5 + end + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_array_join_returns_string() { + eval(r#" + def test = [].join "," + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, join@0x1010, cme:0x1018) + v23:StringExact = CCallVariadic v10, :Array#join@0x1040, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_string_to_s_returns_string() { + eval(r#" + def test = "".to_s + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_string_literal_to_s() { + eval(r#" + def test = "foo".to_s + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_profiled_string_to_s() { + eval(r#" + def test(o) = o.to_s + test "foo" + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, to_s@0x1008, cme:0x1010) + v22:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fixnum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test 5 + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, to_s@0x1008, cme:0x1010) + v21:Fixnum = GuardType v9, Fixnum + v22:StringExact = CCallVariadic v21, :Integer#to_s@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_bignum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test (2**65) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, to_s@0x1008, cme:0x1010) + v21:Integer = GuardType v9, Integer + v22:StringExact = CCallVariadic v21, :Integer#to_s@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_any_to_string_with_known_string_exact() { + eval(r##" + def test(x) = "#{x}" + test 123 + "##); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:Fixnum = GuardType v9, Fixnum + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v30:StringExact = CCallVariadic v26, :Integer#to_s@0x1040 + v21:StringExact = StringConcat v13, v30 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_array_aref_fixnum_literal() { + eval(" + def test + arr = [1, 2, 3] + arr[0] + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v19:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v30:CInt64[0] = UnboxFixnum v19 + v31:CInt64 = ArrayLength v14 + v32:CInt64[0] = GuardLess v30, v31 + v33:CInt64[0] = Const CInt64(0) + v34:CInt64[0] = GuardGreaterEq v32, v33 + v35:BasicObject = ArrayAref v14, v34 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_array_aref_fixnum_profiled() { + eval(" + def test(arr, idx) + arr[idx] + end + test([1, 2, 3], 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@5 + v3:BasicObject = GetLocal :idx, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []@0x1008, cme:0x1010) + v27:ArrayExact = GuardType v11, ArrayExact + v28:Fixnum = GuardType v12, Fixnum + v29:CInt64 = UnboxFixnum v28 + v30:CInt64 = ArrayLength v27 + v31:CInt64 = GuardLess v29, v30 + v32:CInt64[0] = Const CInt64(0) + v33:CInt64 = GuardGreaterEq v31, v32 + v34:BasicObject = ArrayAref v27, v33 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_array_aref_fixnum_array_subclass() { + eval(" + class C < Array; end + def test(arr, idx) + arr[idx] + end + test(C.new([1, 2, 3]), 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@5 + v3:BasicObject = GetLocal :idx, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, []@0x1008, cme:0x1010) + v27:ArraySubclass[class_exact:C] = GuardType v11, ArraySubclass[class_exact:C] + v28:Fixnum = GuardType v12, Fixnum + v29:CInt64 = UnboxFixnum v28 + v30:CInt64 = ArrayLength v27 + v31:CInt64 = GuardLess v29, v30 + v32:CInt64[0] = Const CInt64(0) + v33:CInt64 = GuardGreaterEq v31, v32 + v34:BasicObject = ArrayAref v27, v33 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_hash_aref_literal() { + eval(" + def test + arr = {1 => 3} + arr[1] + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v19:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v30:BasicObject = HashAref v14, v19 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_hash_aref_profiled() { + eval(" + def test(hash, key) + hash[key] + end + test({1 => 3}, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@5 + v3:BasicObject = GetLocal :key, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []@0x1008, cme:0x1010) + v27:HashExact = GuardType v11, HashExact + v28:BasicObject = HashAref v27, v12 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_no_optimize_hash_aref_subclass() { + eval(" + class C < Hash; end + def test(hash, key) + hash[key] + end + test(C.new({0 => 3}), 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@5 + v3:BasicObject = GetLocal :key, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, []@0x1008, cme:0x1010) + v27:HashSubclass[class_exact:C] = GuardType v11, HashSubclass[class_exact:C] + v28:BasicObject = CCallWithFrame v27, :Hash#[]@0x1038, v12 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_does_not_fold_hash_aref_with_frozen_hash() { + eval(" + H = {a: 0}.freeze + def test = H[:a] + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, H) + v23:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:StaticSymbol[:a] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(Hash@0x1018) + PatchPoint MethodRedefined(Hash@0x1018, []@0x1020, cme:0x1028) + v27:BasicObject = HashAref v23, v13 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_hash_aset_literal() { + eval(" + def test + h = {} + h[1] = 3 + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v22:Fixnum[1] = Const Value(1) + v24:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []=@0x1008, cme:0x1010) + HashAset v13, v22, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_hash_aset_profiled() { + eval(" + def test(hash, key, val) + hash[key] = val + end + test({}, 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@6 + v3:BasicObject = GetLocal :key, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []=@0x1008, cme:0x1010) + v35:HashExact = GuardType v13, HashExact + HashAset v35, v14, v15 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_no_optimize_hash_aset_subclass() { + eval(" + class C < Hash; end + def test(hash, key, val) + hash[key] = val + end + test(C.new, 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@6 + v3:BasicObject = GetLocal :key, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, []=@0x1008, cme:0x1010) + v35:HashSubclass[class_exact:C] = GuardType v13, HashSubclass[class_exact:C] + v36:BasicObject = CCallWithFrame v35, :Hash#[]=@0x1038, v14, v15 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_optimize_thread_current() { + eval(" + def test = Thread.current + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Thread) + v20:Class[Thread@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, current@0x1018, cme:0x1020) + v24:CPtr = LoadEC + v25:CPtr = LoadField v24, :thread_ptr@0x1048 + v26:BasicObject = LoadField v25, :self@0x1049 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_array_aset_literal() { + eval(" + def test(arr) + arr[1] = 10 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v16:Fixnum[1] = Const Value(1) + v18:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []=@0x1008, cme:0x1010) + v31:ArrayExact = GuardType v9, ArrayExact + v32:ArrayExact = GuardNotFrozen v31 + v33:ArrayExact = GuardNotShared v32 + v34:CInt64[1] = UnboxFixnum v16 + v35:CInt64 = ArrayLength v33 + v36:CInt64[1] = GuardLess v34, v35 + v37:CInt64[0] = Const CInt64(0) + v38:CInt64[1] = GuardGreaterEq v36, v37 + ArrayAset v33, v38, v18 + WriteBarrier v33, v18 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_optimize_array_aset_profiled() { + eval(" + def test(arr, index, val) + arr[index] = val + end + test([], 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@6 + v3:BasicObject = GetLocal :index, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, []=@0x1008, cme:0x1010) + v35:ArrayExact = GuardType v13, ArrayExact + v36:Fixnum = GuardType v14, Fixnum + v37:ArrayExact = GuardNotFrozen v35 + v38:ArrayExact = GuardNotShared v37 + v39:CInt64 = UnboxFixnum v36 + v40:CInt64 = ArrayLength v38 + v41:CInt64 = GuardLess v39, v40 + v42:CInt64[0] = Const CInt64(0) + v43:CInt64 = GuardGreaterEq v41, v42 + ArrayAset v38, v43, v15 + WriteBarrier v38, v15 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_optimize_array_aset_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, index, val) + arr[index] = val + end + a = MyArray.new + test(a, 0, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@6 + v3:BasicObject = GetLocal :index, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(MyArray@0x1000) + PatchPoint MethodRedefined(MyArray@0x1000, []=@0x1008, cme:0x1010) + v35:ArraySubclass[class_exact:MyArray] = GuardType v13, ArraySubclass[class_exact:MyArray] + v36:BasicObject = CCallVariadic v35, :Array#[]=@0x1038, v14, v15 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_optimize_array_ltlt() { + eval(" + def test(arr) + arr << 1 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, <<@0x1008, cme:0x1010) + v25:ArrayExact = GuardType v9, ArrayExact + ArrayPush v25, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_array_push_single_arg() { + eval(" + def test(arr) + arr.push(1) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, push@0x1008, cme:0x1010) + v24:ArrayExact = GuardType v9, ArrayExact + ArrayPush v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_optimize_array_push_multi_arg() { + eval(" + def test(arr) + arr.push(1,2,3) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + v16:Fixnum[2] = Const Value(2) + v18:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, push@0x1008, cme:0x1010) + v28:ArrayExact = GuardType v9, ArrayExact + v29:BasicObject = CCallVariadic v28, :Array#push@0x1038, v14, v16, v18 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_array_length() { + eval(" + def test(arr) = arr.length + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:CInt64 = ArrayLength v23 + v25:Fixnum = BoxFixnum v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_array_size() { + eval(" + def test(arr) = arr.size + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arr, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + v23:ArrayExact = GuardType v9, ArrayExact + v24:CInt64 = ArrayLength v23 + v25:Fixnum = BoxFixnum v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_optimize_regexpmatch2() { + eval(r#" + def test(s) = s =~ /a/ + test("foo") + "#); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:RegexpExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, =~@0x1010, cme:0x1018) + v25:StringExact = GuardType v9, StringExact + v26:BasicObject = CCallWithFrame v25, :String#=~@0x1040, v14 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_string_getbyte_fixnum() { + eval(r#" + def test(s, i) = s.getbyte(i) + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@5 + v3:BasicObject = GetLocal :i, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, getbyte@0x1008, cme:0x1010) + v26:StringExact = GuardType v11, StringExact + v27:Fixnum = GuardType v12, Fixnum + v28:CInt64 = UnboxFixnum v27 + v29:CInt64 = LoadField v26, :len@0x1038 + v30:CInt64 = GuardLess v28, v29 + v31:CInt64[0] = Const CInt64(0) + v32:CInt64 = GuardGreaterEq v30, v31 + v33:Fixnum = StringGetbyte v26, v30 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_elide_string_getbyte_fixnum() { + eval(r#" + def test(s, i) + s.getbyte(i) + 5 + end + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@5 + v3:BasicObject = GetLocal :i, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, getbyte@0x1008, cme:0x1010) + v30:StringExact = GuardType v11, StringExact + v31:Fixnum = GuardType v12, Fixnum + v32:CInt64 = UnboxFixnum v31 + v33:CInt64 = LoadField v30, :len@0x1038 + v34:CInt64 = GuardLess v32, v33 + v35:CInt64[0] = Const CInt64(0) + v36:CInt64 = GuardGreaterEq v34, v35 + IncrCounter inline_cfunc_optimized_send_count + v22:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_string_setbyte_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 127) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@6 + v3:BasicObject = GetLocal :idx, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, setbyte@0x1008, cme:0x1010) + v30:StringExact = GuardType v13, StringExact + v31:Fixnum = GuardType v14, Fixnum + v32:Fixnum = GuardType v15, Fixnum + v33:CInt64 = UnboxFixnum v31 + v34:CInt64 = LoadField v30, :len@0x1038 + v35:CInt64 = GuardLess v33, v34 + v36:CInt64[0] = Const CInt64(0) + v37:CInt64 = GuardGreaterEq v35, v36 + v38:StringExact = GuardNotFrozen v30 + v39:Fixnum = StringSetbyteFixnum v38, v31, v32 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_string_subclass_setbyte_fixnum() { + eval(r#" + class MyString < String + end + def test(s, idx, val) + s.setbyte(idx, val) + end + test(MyString.new('foo'), 0, 127) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@6 + v3:BasicObject = GetLocal :idx, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1000) + PatchPoint MethodRedefined(MyString@0x1000, setbyte@0x1008, cme:0x1010) + v30:StringSubclass[class_exact:MyString] = GuardType v13, StringSubclass[class_exact:MyString] + v31:Fixnum = GuardType v14, Fixnum + v32:Fixnum = GuardType v15, Fixnum + v33:CInt64 = UnboxFixnum v31 + v34:CInt64 = LoadField v30, :len@0x1038 + v35:CInt64 = GuardLess v33, v34 + v36:CInt64[0] = Const CInt64(0) + v37:CInt64 = GuardGreaterEq v35, v36 + v38:StringSubclass[class_exact:MyString] = GuardNotFrozen v30 + v39:Fixnum = StringSetbyteFixnum v38, v31, v32 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_do_not_optimize_string_setbyte_non_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 3.14) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@6 + v3:BasicObject = GetLocal :idx, l0, SP@5 + v4:BasicObject = GetLocal :val, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, setbyte@0x1008, cme:0x1010) + v30:StringExact = GuardType v13, StringExact + v31:BasicObject = CCallWithFrame v30, :String#setbyte@0x1038, v14, v15 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_specialize_string_empty() { + eval(r#" + def test(s) + s.empty? + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, empty?@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + v24:CInt64 = LoadField v23, :len@0x1038 + v25:CInt64[0] = Const CInt64(0) + v26:CBool = IsBitEqual v24, v25 + v27:BoolExact = BoxBool v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_eliminate_string_empty() { + eval(r#" + def test(s) + s.empty? + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, empty?@0x1008, cme:0x1010) + v27:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_inline_integer_succ_with_fixnum() { + eval(" + def test(x) = x.succ + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, succ@0x1008, cme:0x1010) + v22:Fixnum = GuardType v9, Fixnum + v23:Fixnum[1] = Const Value(1) + v24:Fixnum = FixnumAdd v22, v23 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_bignum() { + eval(" + def test(x) = x.succ + test(4 << 70) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, succ@0x1008, cme:0x1010) + v22:Integer = GuardType v9, Integer + v23:BasicObject = CCallWithFrame v22, :Integer#succ@0x1038 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_inline_integer_ltlt_with_known_fixnum() { + eval(" + def test(x) = x << 5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:Fixnum = FixnumLShift v24, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_negative() { + eval(" + def test(x) = x << -5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BasicObject = CCallWithFrame v24, :Integer#<<@0x1038, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_out_of_range() { + eval(" + def test(x) = x << 64 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v24:Fixnum = GuardType v9, Fixnum + v25:BasicObject = CCallWithFrame v24, :Integer#<<@0x1038, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_unknown_fixnum() { + eval(" + def test(x, y) = x << y + test(4, 5) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, <<@0x1008, cme:0x1010) + v26:Fixnum = GuardType v11, Fixnum + v27:BasicObject = CCallWithFrame v26, :Integer#<<@0x1038, v12 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_integer_gtgt_with_known_fixnum() { + eval(" + def test(x) = x >> 5 + test(4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v23:Fixnum = GuardType v9, Fixnum + v24:Fixnum = FixnumRShift v23, v14 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_negative() { + eval(" + def test(x) = x >> -5 + test(4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v23:Fixnum = GuardType v9, Fixnum + v24:BasicObject = CCallWithFrame v23, :Integer#>>@0x1038, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_out_of_range() { + eval(" + def test(x) = x >> 64 + test(4) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v23:Fixnum = GuardType v9, Fixnum + v24:BasicObject = CCallWithFrame v23, :Integer#>>@0x1038, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_unknown_fixnum() { + eval(" + def test(x, y) = x >> y + test(4, 5) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, >>@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + v26:BasicObject = CCallWithFrame v25, :Integer#>>@0x1038, v12 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_string_append() { + eval(r#" + def test(x, y) = x << y + test("iron", "fish") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, <<@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:StringExact = StringAppend v27, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_string_append_codepoint() { + eval(r#" + def test(x, y) = x << y + test("iron", 4) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, <<@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:Fixnum = GuardType v12, Fixnum + v29:StringExact = StringAppendCodepoint v27, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_string_append_string_subclass() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test("iron", MyString.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, <<@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:StringExact = StringAppend v27, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_do_not_optimize_string_subclass_append_string() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test(MyString.new, "iron") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1000) + PatchPoint MethodRedefined(MyString@0x1000, <<@0x1008, cme:0x1010) + v27:StringSubclass[class_exact:MyString] = GuardType v11, StringSubclass[class_exact:MyString] + v28:BasicObject = CCallWithFrame v27, :String#<<@0x1038, v12 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_optimize_string_append_non_string() { + eval(r#" + def test = "iron" << :a + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, <<@0x1018, cme:0x1020) + v24:BasicObject = CCallWithFrame v11, :String#<<@0x1048, v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_many_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo(3, 3, 3) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[3] = Const Value(3) + v14:Fixnum[3] = Const Value(3) + v16:Fixnum[3] = Const Value(3) + v18:BasicObject = SendWithoutBlock v10, :foo, v12, v14, v16 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_optimize_string_ascii_only_p() { + eval(r#" + def test(x) = x.ascii_only? + test("iron") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ascii_only?@0x1008, cme:0x1010) + v22:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v24:BoolExact = CCall v22, :String#ascii_only?@0x1038 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_few_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:BasicObject = SendWithoutBlock v10, :foo # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_args() { + eval(" + def test = 4.succ 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[1] = Const Value(1) + v14:BasicObject = SendWithoutBlock v10, :succ, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_integer_xor_with_fixnum() { + eval(" + def test(x, y) = x ^ y + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + v26:Fixnum = GuardType v12, Fixnum + v27:Fixnum = FixnumXor v25, v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_eliminate_integer_xor() { + eval(r#" + def test(x, y) + x ^ y + 42 + end + test(1, 2) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v29:Fixnum = GuardType v11, Fixnum + v30:Fixnum = GuardType v12, Fixnum + IncrCounter inline_cfunc_optimized_send_count + v22:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_bignum_or_boolean() { + eval(" + def test(x, y) = x ^ y + test(4 << 70, 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v25:Integer = GuardType v11, Integer + v26:BasicObject = CCallWithFrame v25, :Integer#^@0x1038, v12 + CheckInterrupts + Return v26 + "); + + eval(" + def test(x, y) = x ^ y + test(1, 4 << 70) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v25:Fixnum = GuardType v11, Fixnum + v26:BasicObject = CCallWithFrame v25, :Integer#^@0x1038, v12 + CheckInterrupts + Return v26 + "); + + eval(" + def test(x, y) = x ^ y + test(true, 0) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1000, ^@0x1008, cme:0x1010) + v25:TrueClass = GuardType v11, TrueClass + v26:BasicObject = CCallWithFrame v25, :TrueClass#^@0x1038, v12 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_args() { + eval(" + def test(x, y) = x.^() + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v17:BasicObject = SendWithoutBlock v11, :^ # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_specialize_hash_size() { + eval(" + def test(hash) = hash.size + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, size@0x1008, cme:0x1010) + v23:HashExact = GuardType v9, HashExact + IncrCounter inline_cfunc_optimized_send_count + v25:Fixnum = CCall v23, :Hash#size@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_eliminate_hash_size() { + eval(" + def test(hash) + hash.size + 5 + end + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :hash, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, size@0x1008, cme:0x1010) + v27:HashExact = GuardType v9, HashExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_respond_to_p_true() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v28:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_no_method() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint MethodRedefined(C@0x1008, respond_to_missing?@0x1040, cme:0x1048) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1070, cme:0x1078) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_default_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v28:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, false) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:FalseClass = Const Value(false) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_true_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, true) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:TrueClass = Const Value(true) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_truthy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, 4) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + v16:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v26:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1040, cme:0x1048) + v30:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_missing() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + PatchPoint MethodRedefined(C@0x1008, respond_to_missing?@0x1040, cme:0x1048) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1070, cme:0x1078) + v30:FalseClass = Const Value(false) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_do_not_optimize_redefined_respond_to_missing() { + eval(r#" + class C + def respond_to_missing?(method, include_private = false) + true + end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:7: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, respond_to?@0x1010, cme:0x1018) + v24:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + v25:BasicObject = CCallVariadic v24, :Kernel#respond_to?@0x1040, v14 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putself() { + eval(r#" + def callee = self + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_string() { + eval(r#" + # frozen_string_literal: true + def callee = "abc" + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:StringExact[VALUE(0x1038)] = Const Value(VALUE(0x1038)) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putnil() { + eval(r#" + def callee = nil + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:NilClass = Const Value(nil) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_true() { + eval(r#" + def callee = true + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:TrueClass = Const Value(true) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_false() { + eval(r#" + def callee = false + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:FalseClass = Const Value(false) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_zero() { + eval(r#" + def callee = 0 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_one() { + eval(r#" + def callee = 1 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_send_without_block_direct_parameter() { + eval(r#" + def callee(x) = x + def test = callee 3 + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v20:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_without_block_direct_last_parameter() { + eval(r#" + def callee(x, y, z) = z + def test = callee 1, 2, 3 + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v24:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_splat() { + eval(" + def foo = itself + + def test + # Use a local to inhibit compile.c peephole optimization to ensure callsites have VM_CALL_ARGS_SPLAT + empty = [] + foo(*empty) + ''.display(*empty) + itself(*empty) + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v19:ArrayExact = ToArray v13 + IncrCounter complex_arg_pass_caller_splat + v21:BasicObject = SendWithoutBlock v8, :foo, v19 # SendFallbackReason: Complex argument passing + v25:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:StringExact = StringCopy v25 + PatchPoint NoEPEscape(test) + v31:ArrayExact = ToArray v13 + IncrCounter complex_arg_pass_caller_splat + v33:BasicObject = SendWithoutBlock v26, :display, v31 # SendFallbackReason: Complex argument passing + PatchPoint NoEPEscape(test) + v41:ArrayExact = ToArray v13 + IncrCounter complex_arg_pass_caller_splat + v43:BasicObject = SendWithoutBlock v8, :itself, v41 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_inline_symbol_to_sym() { + eval(r#" + def test(o) = o.to_sym + test :foo + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1000, to_sym@0x1008, cme:0x1010) + v20:StaticSymbol = GuardType v9, StaticSymbol + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_integer_to_i() { + eval(r#" + def test(o) = o.to_i + test 5 + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(Integer@0x1000, to_i@0x1008, cme:0x1010) + v20:Fixnum = GuardType v9, Fixnum + IncrCounter inline_iseq_optimized_send_count + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_stringexact_eq_stringexact() { + eval(r#" + def test(l, r) = l == r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ==@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:BoolExact = CCall v27, :String#==@0x1038, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_string_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, ==@0x1008, cme:0x1010) + v27:StringSubclass[class_exact:C] = GuardType v11, StringSubclass[class_exact:C] + v28:String = GuardType v12, String + v29:BoolExact = CCall v27, :String#==@0x1038, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_stringexact_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ==@0x1008, cme:0x1010) + v27:StringExact = GuardType v11, StringExact + v28:String = GuardType v12, String + v29:BoolExact = CCall v27, :String#==@0x1038, v28 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_stringexact() { + eval(r#" + def test(l, r) = l === r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ===@0x1008, cme:0x1010) + v26:StringExact = GuardType v11, StringExact + v27:String = GuardType v12, String + v28:BoolExact = CCall v26, :String#==@0x1038, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_string_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, ===@0x1008, cme:0x1010) + v26:StringSubclass[class_exact:C] = GuardType v11, StringSubclass[class_exact:C] + v27:String = GuardType v12, String + v28:BoolExact = CCall v26, :String#==@0x1038, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :l, l0, SP@5 + v3:BasicObject = GetLocal :r, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, ===@0x1008, cme:0x1010) + v26:StringExact = GuardType v11, StringExact + v27:String = GuardType v12, String + v28:BoolExact = CCall v26, :String#==@0x1038, v27 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_specialize_string_size() { + eval(r#" + def test(s) + s.size + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, size@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v25:Fixnum = CCall v23, :String#size@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_elide_string_size() { + eval(r#" + def test(s) + s.size + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, size@0x1008, cme:0x1010) + v27:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_inline_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, bytesize@0x1008, cme:0x1010) + v22:StringExact = GuardType v9, StringExact + v23:CInt64 = LoadField v22, :len@0x1038 + v24:Fixnum = BoxFixnum v23 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_elide_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, bytesize@0x1008, cme:0x1010) + v26:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_specialize_string_length() { + eval(r#" + def test(s) + s.length + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, length@0x1008, cme:0x1010) + v23:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v25:Fixnum = CCall v23, :String#length@0x1038 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_class_eqq() { + eval(r#" + def test(o) = String === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v26:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, ===@0x1018, cme:0x1020) + v30:BoolExact = IsA v9, v26 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_specialize_module_eqq() { + eval(r#" + def test(o) = Kernel === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v26:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, ===@0x1018, cme:0x1020) + IncrCounter inline_cfunc_optimized_send_count + v31:BoolExact = CCall v26, :Module#===@0x1048, v9 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_specialize_is_a_class() { + eval(r#" + def test(o) = o.is_a?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v24:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, is_a?@0x1009, cme:0x1010) + v28:StringExact = GuardType v9, StringExact + v29:BoolExact = IsA v28, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_dont_specialize_is_a_module() { + eval(r#" + def test(o) = o.is_a?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v24:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, is_a?@0x1018, cme:0x1020) + v28:StringExact = GuardType v9, StringExact + v29:BasicObject = CCallWithFrame v28, :Kernel#is_a?@0x1048, v24 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_elide_is_a() { + eval(r#" + def test(o) + o.is_a?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v28:Class[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, is_a?@0x1018, cme:0x1020) + v32:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_elide_class_eqq() { + eval(r#" + def test(o) + Integer === o + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v30:Class[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, ===@0x1018, cme:0x1020) + IncrCounter inline_cfunc_optimized_send_count + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_kind_of_class() { + eval(r#" + def test(o) = o.kind_of?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v24:Class[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, kind_of?@0x1009, cme:0x1010) + v28:StringExact = GuardType v9, StringExact + v29:BoolExact = IsA v28, v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_dont_specialize_kind_of_module() { + eval(r#" + def test(o) = o.kind_of?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v24:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, kind_of?@0x1018, cme:0x1020) + v28:StringExact = GuardType v9, StringExact + v29:BasicObject = CCallWithFrame v28, :Kernel#kind_of?@0x1048, v24 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_elide_kind_of() { + eval(r#" + def test(o) + o.kind_of?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v28:Class[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, kind_of?@0x1018, cme:0x1020) + v32:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn counting_complex_feature_use_for_fallback() { + eval(" + define_method(:fancy) { |_a, *_b, kw: 100, **kw_rest, &block| } + def test = fancy(1) + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + IncrCounter complex_arg_pass_param_rest + IncrCounter complex_arg_pass_param_block + IncrCounter complex_arg_pass_param_kwrest + IncrCounter complex_arg_pass_param_kw_opt + v13:BasicObject = SendWithoutBlock v6, :fancy, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn call_method_forwardable_param() { + eval(" + def forwardable(...) = itself(...) + def call_forwardable = forwardable + call_forwardable + "); + assert_snapshot!(hir_string("call_forwardable"), @r" + fn call_forwardable@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + IncrCounter complex_arg_pass_param_forwardable + v11:BasicObject = SendWithoutBlock v6, :forwardable # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_string_length() { + eval(r#" + def test(s) + s.length + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(String@0x1000) + PatchPoint MethodRedefined(String@0x1000, length@0x1008, cme:0x1010) + v27:StringExact = GuardType v9, StringExact + IncrCounter inline_cfunc_optimized_send_count + v19:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_fold_self_class_respond_to_true() { + eval(r#" + class C + class << self + attr_accessor :_lex_actions + private :_lex_actions, :_lex_actions= + end + self._lex_actions = [1, 2, 3] + def initialize + if self.class.respond_to?(:_lex_actions, true) + :CORRECT + else + :oh_no_wrong + end + end + end + C.new # warm up + TEST = C.instance_method(:initialize) + "#); + assert_snapshot!(hir_string_proc("TEST"), @r" + fn initialize@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v40:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v44:Class[C@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + v13:StaticSymbol[:_lex_actions] = Const Value(VALUE(0x1038)) + v15:TrueClass = Const Value(true) + PatchPoint NoSingletonClass(Class@0x1040) + PatchPoint MethodRedefined(Class@0x1040, respond_to?@0x1048, cme:0x1050) + PatchPoint NoSingletonClass(Class@0x1040) + PatchPoint MethodRedefined(Class@0x1040, _lex_actions@0x1078, cme:0x1080) + v52:TrueClass = Const Value(true) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + v24:StaticSymbol[:CORRECT] = Const Value(VALUE(0x10a8)) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_self_class_name() { + eval(r#" + class C; end + def test(o) = o.class.name + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v23:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v27:Class[C@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + PatchPoint NoSingletonClass(Class@0x1038) + PatchPoint MethodRedefined(Class@0x1038, name@0x1040, cme:0x1048) + IncrCounter inline_cfunc_optimized_send_count + v33:StringExact|NilClass = CCall v27, :Module#name@0x1070 + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_kernel_class() { + eval(r#" + class C; end + def test(o) = o.class + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v21:HeapObject[class_exact:C] = GuardType v9, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v25:Class[C@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_class() { + eval(r#" + def test = 5.class + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, class@0x1008, cme:0x1010) + IncrCounter inline_iseq_optimized_send_count + v21:Class[Integer@0x1000] = Const Value(VALUE(0x1000)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_fold_singleton_class() { + eval(r#" + def test = self.class + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(Object@0x1000) + PatchPoint MethodRedefined(Object@0x1000, class@0x1008, cme:0x1010) + v18:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1000)] + IncrCounter inline_iseq_optimized_send_count + v22:Class[Object@0x1038] = Const Value(VALUE(0x1038)) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v22 + "); + } + + #[test] + fn no_load_from_ep_right_after_entrypoint() { + let formatted = eval(" + def read_nil_local(a, _b, _c) + formatted ||= a + @formatted = formatted + -> { formatted } # the environment escapes + end + + def call + puts [], [], [], [] # fill VM stack with junk + read_nil_local(true, 1, 1) # expected direct send + end + + call # profile + call # compile + @formatted + "); + assert_eq!(Qtrue, formatted, "{}", formatted.obj_info()); + assert_snapshot!(hir_string("read_nil_local"), @r" + fn read_nil_local@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :_b, l0, SP@6 + v4:BasicObject = GetLocal :_c, l0, SP@5 + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject): + EntryPoint JIT(0) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass): + CheckInterrupts + SetLocal :formatted, l0, EP@3, v15 + PatchPoint SingleRactorMode + v54:HeapBasicObject = GuardType v14, HeapBasicObject + v55:CShape = LoadField v54, :_shape_id@0x1000 + v56:CShape[0x1001] = GuardBitEquals v55, CShape(0x1001) + StoreField v54, :@formatted@0x1002, v15 + WriteBarrier v54, v15 + v59:CShape[0x1003] = Const CShape(0x1003) + StoreField v54, :_shape_id@0x1000, v59 + v43:Class[VMFrozenCore] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Class@0x1010) + PatchPoint MethodRedefined(Class@0x1010, lambda@0x1018, cme:0x1020) + v64:BasicObject = CCallWithFrame v43, :RubyVM::FrozenCore.lambda@0x1048, block=0x1050 + v46:BasicObject = GetLocal :a, l0, EP@6 + v47:BasicObject = GetLocal :_b, l0, EP@5 + v48:BasicObject = GetLocal :_c, l0, EP@4 + v49:BasicObject = GetLocal :formatted, l0, EP@3 + CheckInterrupts + Return v64 + "); + } + + #[test] + fn test_fold_load_field_frozen_constant_object() { + // Basic case: frozen constant object with attr_accessor + eval(" + class TestFrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + FROZEN_OBJ = TestFrozen.new.freeze + + def test = FROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_OBJ) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozen@0x1010) + PatchPoint MethodRedefined(TestFrozen@0x1010, a@0x1018, cme:0x1020) + v29:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_multiple_ivars() { + // Frozen object with multiple instance variables + eval(" + class TestMultiIvars + attr_accessor :a, :b, :c + def initialize + @a = 10 + @b = 20 + @c = 30 + end + end + + MULTI_FROZEN = TestMultiIvars.new.freeze + + def test = MULTI_FROZEN.b + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:13: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MULTI_FROZEN) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestMultiIvars@0x1010) + PatchPoint MethodRedefined(TestMultiIvars@0x1010, b@0x1018, cme:0x1020) + v29:Fixnum[20] = Const Value(20) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_string_value() { + // Frozen object with a string ivar + eval(r#" + class TestFrozenStr + attr_accessor :name + def initialize + @name = "hello" + end + end + + FROZEN_STR = TestFrozenStr.new.freeze + + def test = FROZEN_STR.name + test + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_STR) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenStr@0x1010) + PatchPoint MethodRedefined(TestFrozenStr@0x1010, name@0x1018, cme:0x1020) + v29:StringExact[VALUE(0x1048)] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_nil_value() { + // Frozen object with nil ivar + eval(" + class TestFrozenNil + attr_accessor :value + def initialize + @value = nil + end + end + + FROZEN_NIL = TestFrozenNil.new.freeze + + def test = FROZEN_NIL.value + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_NIL) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenNil@0x1010) + PatchPoint MethodRedefined(TestFrozenNil@0x1010, value@0x1018, cme:0x1020) + v29:NilClass = Const Value(nil) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_no_fold_load_field_unfrozen_object() { + // Non-frozen object should NOT be folded + eval(" + class TestUnfrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + UNFROZEN_OBJ = TestUnfrozen.new + + def test = UNFROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, UNFROZEN_OBJ) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestUnfrozen@0x1010) + PatchPoint MethodRedefined(TestUnfrozen@0x1010, a@0x1018, cme:0x1020) + v25:CShape = LoadField v20, :_shape_id@0x1048 + v26:CShape[0x1049] = GuardBitEquals v25, CShape(0x1049) + v27:BasicObject = LoadField v20, :@a@0x104a + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_load_field_frozen_with_attr_reader() { + // Using attr_reader instead of attr_accessor + eval(" + class TestAttrReader + attr_reader :value + def initialize(v) + @value = v + end + end + + FROZEN_READER = TestAttrReader.new(42).freeze + + def test = FROZEN_READER.value + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_READER) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestAttrReader@0x1010) + PatchPoint MethodRedefined(TestAttrReader@0x1010, value@0x1018, cme:0x1020) + v29:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_symbol_value() { + // Frozen object with a symbol ivar + eval(" + class TestFrozenSym + attr_accessor :sym + def initialize + @sym = :hello + end + end + + FROZEN_SYM = TestFrozenSym.new.freeze + + def test = FROZEN_SYM.sym + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_SYM) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenSym@0x1010) + PatchPoint MethodRedefined(TestFrozenSym@0x1010, sym@0x1018, cme:0x1020) + v29:StaticSymbol[:hello] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_load_field_frozen_true_false() { + // Frozen object with boolean ivars + eval(" + class TestFrozenBool + attr_accessor :flag + def initialize + @flag = true + end + end + + FROZEN_TRUE = TestFrozenBool.new.freeze + + def test = FROZEN_TRUE.flag + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:11: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_TRUE) + v20:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenBool@0x1010) + PatchPoint MethodRedefined(TestFrozenBool@0x1010, flag@0x1018, cme:0x1020) + v29:TrueClass = Const Value(true) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_no_fold_load_field_dynamic_receiver() { + // Dynamic receiver (not a constant) should NOT be folded even if object is frozen + eval(" + class TestDynamic + attr_accessor :val + def initialize + @val = 99 + end + end + + def test(obj) = obj.val + o = TestDynamic.new.freeze + test o + test o + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :obj, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint NoSingletonClass(TestDynamic@0x1000) + PatchPoint MethodRedefined(TestDynamic@0x1000, val@0x1008, cme:0x1010) + v21:HeapObject[class_exact:TestDynamic] = GuardType v9, HeapObject[class_exact:TestDynamic] + v24:CShape = LoadField v21, :_shape_id@0x1038 + v25:CShape[0x1039] = GuardBitEquals v24, CShape(0x1039) + v26:BasicObject = LoadField v21, :@val@0x103a + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_load_field_frozen_nested_access() { + // Accessing multiple fields from frozen constant in sequence + eval(" + class TestNestedAccess + attr_accessor :x, :y + def initialize + @x = 100 + @y = 200 + end + end + + NESTED_FROZEN = TestNestedAccess.new.freeze + + def test = NESTED_FROZEN.x + NESTED_FROZEN.y + test + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:12: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, NESTED_FROZEN) + v28:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestNestedAccess@0x1010) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, x@0x1018, cme:0x1020) + v53:Fixnum[100] = Const Value(100) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1048, NESTED_FROZEN) + v34:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestNestedAccess@0x1010) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, y@0x1050, cme:0x1058) + v55:Fixnum[200] = Const Value(200) + PatchPoint MethodRedefined(Integer@0x1080, +@0x1088, cme:0x1090) + v56:Fixnum[300] = Const Value(300) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v56 + "); + } + + #[test] + fn test_dont_fold_load_field_with_primitive_return_type() { + eval(r#" + S = "abc".freeze + def test = S.bytesize + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v20:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, bytesize@0x1018, cme:0x1020) + v24:CInt64 = LoadField v20, :len@0x1048 + v25:Fixnum = BoxFixnum v24 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v25 + "); + } + + #[test] + fn optimize_call_to_private_method_iseq_with_fcall() { + eval(r#" + class C + def callprivate = secret + private def secret = 42 + end + C.new.callprivate + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprivate)"), @r" + fn callprivate@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v18:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_iseq() { + eval(r#" + class C + private def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + #[test] + fn optimize_call_to_private_method_cfunc_with_fcall() { + eval(r#" + class BasicObject + def callprivate = initialize rescue $! + end + Obj = BasicObject.new.callprivate + "#); + assert_snapshot!(hir_string_proc("BasicObject.instance_method(:callprivate)"), @r" + fn callprivate@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(BasicObject@0x1000) + PatchPoint MethodRedefined(BasicObject@0x1000, initialize@0x1008, cme:0x1010) + v20:BasicObjectExact = GuardType v6, BasicObjectExact + v21:NilClass = Const Value(nil) + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_cfunc() { + eval(r#" + Obj = BasicObject.new + def test = Obj.initialize rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:BasicObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :initialize # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dont_optimize_call_to_private_top_level_method() { + eval(r#" + def toplevel_method = :OK + Obj = Object.new + def test = Obj.toplevel_method rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :toplevel_method # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + #[test] + fn optimize_call_to_protected_method_iseq_with_fcall() { + eval(r#" + class C + def callprotected = secret + protected def secret = 42 + end + C.new.callprotected + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprotected)"), @r" + fn callprotected@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v18:HeapObject[class_exact:C] = GuardType v6, HeapObject[class_exact:C] + IncrCounter inline_iseq_optimized_send_count + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_protected_method_iseq() { + eval(r#" + class C + protected def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:HeapObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v13:BasicObject = SendWithoutBlock v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v13 + "); + } + + // Test that when a singleton class has been seen for a class, we skip the + // NoSingletonClass optimization to avoid an invalidation loop. + #[test] + fn test_skip_optimization_after_singleton_class_seen() { + // First, trigger the singleton class callback for String by creating a singleton class. + // This should mark String as having had a singleton class seen. + eval(r#" + "hello".singleton_class + "#); + + // Now define and compile a method that would normally be optimized with NoSingletonClass. + // Since String has had a singleton class, the optimization should be skipped and we + // should fall back to SendWithoutBlock. + eval(r#" + def test(s) + s.length + end + test("asdf") + "#); + + // The output should NOT have NoSingletonClass patchpoint for String, and should + // fall back to SendWithoutBlock instead of the optimized CCall path. + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :s, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :length # SendFallbackReason: Singleton class previously created for receiver class + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_invokesuper_to_iseq_optimizes_to_direct() { + eval(" + class A + def foo + 'A' + end + end + + class B < A + def foo + super + end + end + + B.new.foo; B.new.foo + "); + + // A Ruby method as the target of `super` should optimize provided no block is given. + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendWithoutBlockDirect but got:\n{hir}"); + assert!(hir.contains("SendWithoutBlockDirect"), "Should optimize to SendWithoutBlockDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) + GuardSuperMethodEntry 0x1038 + v18:RubyValue = GetBlockHandler + v19:FalseClass = GuardBitEquals v18, Value(false) + v20:BasicObject = SendWithoutBlockDirect v6, :foo (0x1040) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_invokesuper_with_positional_args_optimizes_to_direct() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(x) + super(x) + 1 + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendWithoutBlockDirect but got:\n{hir}"); + assert!(hir.contains("SendWithoutBlockDirect"), "Should optimize to SendWithoutBlockDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) + GuardSuperMethodEntry 0x1038 + v27:RubyValue = GetBlockHandler + v28:FalseClass = GuardBitEquals v27, Value(false) + v29:BasicObject = SendWithoutBlockDirect v8, :foo (0x1040), v9 + v17:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) + v32:Fixnum = GuardType v29, Fixnum + v33:Fixnum = FixnumAdd v32, v17 + IncrCounter inline_cfunc_optimized_send_count + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_invokesuper_with_forwarded_splat_args_remains_invokesuper() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(*x) + super + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :x, l0, SP@4, * + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:ArrayExact): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:ArrayExact): + v15:ArrayExact = ToArray v9 + v17:BasicObject = InvokeSuper v8, 0x1000, v15 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_invokesuper_with_block_literal_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo + super { 'from subclass' } + end + end + + B.new.foo; B.new.foo + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for block literal:\n{hir}"); + + // With a block, we don't optimize to SendWithoutBlockDirect + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: call made with a block + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_to_cfunc_remains_invokesuper() { + eval(" + class MyArray < Array + def length + super + end + end + + MyArray.new.length; MyArray.new.length + "); + + let hir = hir_string_proc("MyArray.new.method(:length)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for CFUNC:\n{hir}"); + + assert_snapshot!(hir, @r" + fn length@<compiled>:4: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: unsupported target method type Cfunc + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_with_blockarg_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo(&blk) + other_block = proc { 'different block' } + super(&other_block) + end + end + + B.new.foo { 'passed' }; B.new.foo { 'passed' } + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :blk, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + PatchPoint NoSingletonClass(B@0x1000) + PatchPoint MethodRedefined(B@0x1000, proc@0x1008, cme:0x1010) + v35:HeapObject[class_exact:B] = GuardType v10, HeapObject[class_exact:B] + v36:BasicObject = CCallWithFrame v35, :Kernel#proc@0x1038, block=0x1040 + v18:BasicObject = GetLocal :blk, l0, EP@4 + SetLocal :other_block, l0, EP@3, v36 + v25:BasicObject = GetLocal :other_block, l0, EP@3 + v27:BasicObject = InvokeSuper v10, 0x1048, v25 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_invokesuper_with_symbol_to_proc_remains_invokesuper() { + eval(" + class A + def foo(items, &blk) + items.map(&blk) + end + end + + class B < A + def foo(items) + super(items, &:succ) + end + end + + B.new.foo([1, 2, 3]); B.new.foo([1, 2, 3]) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendWithoutBlockDirect"), "Should not optimize to SendWithoutBlockDirect for symbol-to-proc:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :items, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:StaticSymbol[:succ] = Const Value(VALUE(0x1000)) + v17:BasicObject = InvokeSuper v8, 0x1008, v9, v15 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_invokesuper_with_keyword_args_remains_invokesuper() { + eval(" + class A + def foo(attributes = {}) + @attributes = attributes + end + end + + class B < A + def foo(content = '') + super(content: content) + end + end + + B.new.foo('image data'); B.new.foo('image data') + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + + assert_snapshot!(hir, @r" + fn foo@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :content, l0, SP@4 + v3:CPtr = LoadPC + v4:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v5:CBool = IsBitEqual v3, v4 + IfTrue v5, bb2(v1, v2) + Jump bb4(v1, v2) + bb1(v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v9, v10) + bb2(v16:BasicObject, v17:BasicObject): + v20:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v21:StringExact = StringCopy v20 + Jump bb4(v16, v21) + bb3(v13:BasicObject, v14:BasicObject): + EntryPoint JIT(1) + Jump bb4(v13, v14) + bb4(v24:BasicObject, v25:BasicObject): + v31:BasicObject = InvokeSuper v24, 0x1018, v25 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v31 + "); + } +} diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs new file mode 100644 index 0000000000..1ce5488f47 --- /dev/null +++ b/zjit/src/hir/tests.rs @@ -0,0 +1,4567 @@ +#[cfg(test)] +use super::*; + +#[cfg(test)] +mod snapshot_tests { + use super::*; + use insta::assert_snapshot; + + #[track_caller] + fn hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[track_caller] + fn optimized_hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(&format!("{}.method(:{})", "self", method))); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v13:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [a=v11, b=v12] } + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [a=v11, b=v12] } + PatchPoint NoTracePoint + v16:Any = Snapshot FrameState { pc: 0x1010, stack: [v11], locals: [a=v11, b=v12] } + v17:Any = Snapshot FrameState { pc: 0x1018, stack: [v11, v12], locals: [a=v11, b=v12] } + v18:ArrayExact = NewArray v11, v12 + v19:Any = Snapshot FrameState { pc: 0x1020, stack: [v18], locals: [a=v11, b=v12] } + PatchPoint NoTracePoint + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_send_direct_with_reordered_kwargs_has_snapshot() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + v16:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15], locals: [] } + PatchPoint NoSingletonClass(Object@0x1010) + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v24:HeapObject[class_exact*:Object@VALUE(0x1010)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1010)] + v25:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v13, v15, v11], locals: [] } + v26:BasicObject = SendWithoutBlockDirect v24, :foo (0x1048), v13, v15, v11 + v18:Any = Snapshot FrameState { pc: 0x1050, stack: [v26], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_with_kwargs_in_order_has_snapshot() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } + PatchPoint NoSingletonClass(Object@0x1010) + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v22:HeapObject[class_exact*:Object@VALUE(0x1010)] = GuardType v6, HeapObject[class_exact*:Object@VALUE(0x1010)] + v23:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } + v24:BasicObject = SendWithoutBlockDirect v22, :foo (0x1048), v11, v13 + v16:Any = Snapshot FrameState { pc: 0x1050, stack: [v24], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_send_direct_with_many_kwargs_no_reorder_snapshot() { + eval(" + def foo(five, six, a:, b:, c:, d:, e:, f:) = [a, b, c, d, five, six, e, f] + def test = foo(5, 6, d: 4, c: 3, a: 1, b: 2, e: 7, f: 8) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[5] = Const Value(5) + v13:Fixnum[6] = Const Value(6) + v15:Fixnum[4] = Const Value(4) + v17:Fixnum[3] = Const Value(3) + v19:Fixnum[1] = Const Value(1) + v21:Fixnum[2] = Const Value(2) + v23:Fixnum[7] = Const Value(7) + v25:Fixnum[8] = Const Value(8) + v26:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15, v17, v19, v21, v23, v25], locals: [] } + v27:BasicObject = SendWithoutBlock v6, :foo, v11, v13, v15, v17, v19, v21, v23, v25 # SendFallbackReason: Too many arguments for LIR + v28:Any = Snapshot FrameState { pc: 0x1010, stack: [v27], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v27 + "); + } +} + +#[cfg(test)] +pub mod hir_build_tests { + use super::*; + use insta::assert_snapshot; + + fn iseq_contains_opcode(iseq: IseqPtr, expected_opcode: u32) -> bool { + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + let mut insn_idx = 0; + while insn_idx < iseq_size { + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + if opcode == expected_opcode { + return true; + } + insn_idx += insn_len(opcode as usize); + } + false + } + + #[track_caller] + pub fn assert_contains_opcode(method: &str, opcode: u32) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + + #[track_caller] + fn assert_contains_opcodes(method: &str, opcodes: &[u32]) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + for &opcode in opcodes { + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + } + + /// Combine multiple hir_string() results to match all of them at once, which allows + /// us to avoid running the set of zjit-test -> zjit-test-update multiple times. + #[macro_export] + macro_rules! hir_strings { + ($( $s:expr ),+ $(,)?) => {{ + vec![$( hir_string($s) ),+].join("\n") + }}; + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + fn assert_compile_fails(method: &str, reason: ParseError) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let result = iseq_to_hir(iseq); + assert!(result.is_err(), "Expected an error but successfully compiled to HIR: {}", FunctionPrinter::without_snapshot(&result.unwrap())); + assert_eq!(result.unwrap_err(), reason); + } + + #[test] + fn test_compile_optional() { + eval("def test(x=1) = 123"); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + v3:CPtr = LoadPC + v4:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v5:CBool = IsBitEqual v3, v4 + IfTrue v5, bb2(v1, v2) + Jump bb4(v1, v2) + bb1(v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v9, v10) + bb2(v16:BasicObject, v17:BasicObject): + v20:Fixnum[1] = Const Value(1) + Jump bb4(v16, v20) + bb3(v13:BasicObject, v14:BasicObject): + EntryPoint JIT(1) + Jump bb4(v13, v14) + bb4(v23:BasicObject, v24:BasicObject): + v28:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_putobject() { + eval("def test = 123"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_array() { + eval("def test = []"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact = NewArray + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_array_with_element() { + eval("def test(a) = [a]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = NewArray v9 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_new_range_inclusive_with_one_element() { + eval("def test(a) = (a..10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v16:RangeExact = NewRange v9 NewRangeInclusive v14 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_new_range_inclusive_with_two_elements() { + eval("def test(a, b) = (a..b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:RangeExact = NewRange v11 NewRangeInclusive v12 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_new_range_exclusive_with_one_element() { + eval("def test(a) = (a...10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[10] = Const Value(10) + v16:RangeExact = NewRange v9 NewRangeExclusive v14 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_new_range_exclusive_with_two_elements() { + eval("def test(a, b) = (a...b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:RangeExact = NewRange v11 NewRangeExclusive v12 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_array_dup() { + eval("def test = [1, 2, 3]"); + assert_contains_opcode("test", YARVINSN_duparray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_hash_dup() { + eval("def test = {a: 1, b: 2}"); + assert_contains_opcode("test", YARVINSN_duphash); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:HashExact = HashDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_new_hash_empty() { + eval("def test = {}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HashExact = NewHash + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_hash_with_elements() { + eval("def test(aval, bval) = {a: aval, b: bval}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :aval, l0, SP@5 + v3:BasicObject = GetLocal :bval, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v16:StaticSymbol[:a] = Const Value(VALUE(0x1000)) + v19:StaticSymbol[:b] = Const Value(VALUE(0x1008)) + v22:HashExact = NewHash v16: v11, v19: v12 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_string_copy() { + eval("def test = \"hello\""); + assert_contains_opcode("test", YARVINSN_putchilledstring); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_bignum() { + eval("def test = 999999999999999999999999999999999999"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Bignum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_flonum() { + eval("def test = 1.5"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Flonum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_heap_float() { + eval("def test = 1.7976931348623157e+308"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HeapFloat[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_static_sym() { + eval("def test = :foo"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_opt_plus() { + eval("def test = 1+2"); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v15:BasicObject = SendWithoutBlock v10, :+, v12 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_opt_hash_freeze() { + eval(" + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_hash_freeze_rewritten() { + eval(" + class Hash + def freeze; 5; end + end + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_ary_freeze() { + eval(" + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_ary_freeze_rewritten() { + eval(" + class Array + def freeze; 5; end + end + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_freeze() { + eval(" + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_freeze_rewritten() { + eval(" + class String + def freeze; 5; end + end + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_uminus() { + eval(" + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_uminus_rewritten() { + eval(" + class String + def -@; 5; end + end + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS)) + "); + } + + #[test] + fn test_setlocal_getlocal() { + eval(" + def test + a = 1 + a + end + "); + assert_contains_opcodes("test", &[YARVINSN_getlocal_WC_0, YARVINSN_setlocal_WC_0]); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_nested_setlocal_getlocal() { + eval(" + l3 = 3 + _unused = _unused1 = nil + 1.times do |l2| + _ = nil + l2 = 2 + 1.times do |l1| + l1 = 1 + define_method(:test) do + l1 = l2 + l2 = l1 + l2 + l3 = l2 + l3 + end + end + end + "); + assert_contains_opcodes( + "test", + &[YARVINSN_getlocal_WC_1, YARVINSN_setlocal_WC_1, + YARVINSN_getlocal, YARVINSN_setlocal]); + assert_snapshot!(hir_string("test"), @r" + fn block (3 levels) in <compiled>@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = GetLocal :l2, l2, EP@4 + SetLocal :l1, l1, EP@3, v10 + v15:BasicObject = GetLocal :l1, l1, EP@3 + v17:BasicObject = GetLocal :l2, l2, EP@4 + v20:BasicObject = SendWithoutBlock v15, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l2, l2, EP@4, v20 + v25:BasicObject = GetLocal :l2, l2, EP@4 + v27:BasicObject = GetLocal :l3, l3, EP@5 + v30:BasicObject = SendWithoutBlock v25, :+, v27 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l3, l3, EP@5, v30 + CheckInterrupts + Return v30 + " + ); + } + + #[test] + fn test_setlocal_in_default_args() { + eval(" + def test(a = (b = 1)) = [a, b] + "); + assert_contains_opcode("test", YARVINSN_setlocal_WC_0); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:NilClass = Const Value(nil) + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v6:CBool = IsBitEqual v4, v5 + IfTrue v6, bb2(v1, v2, v3) + Jump bb4(v1, v2, v3) + bb1(v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v10, v11, v12) + bb2(v19:BasicObject, v20:BasicObject, v21:NilClass): + v25:Fixnum[1] = Const Value(1) + Jump bb4(v19, v25, v25) + bb3(v15:BasicObject, v16:BasicObject): + EntryPoint JIT(1) + v17:NilClass = Const Value(nil) + Jump bb4(v15, v16, v17) + bb4(v30:BasicObject, v31:BasicObject, v32:NilClass|Fixnum): + v38:ArrayExact = NewArray v31, v32 + CheckInterrupts + Return v38 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_tracepoint() { + eval(" + def test(a = (b = 1)) = [a, b] + TracePoint.new(:line) {}.enable + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:NilClass = Const Value(nil) + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v6:CBool = IsBitEqual v4, v5 + IfTrue v6, bb2(v1, v2, v3) + Jump bb4(v1, v2, v3) + bb1(v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v10, v11, v12) + bb2(v19:BasicObject, v20:BasicObject, v21:NilClass): + SideExit UnhandledYARVInsn(trace_putobject_INT2FIX_1_) + bb3(v15:BasicObject, v16:BasicObject): + EntryPoint JIT(1) + v17:NilClass = Const Value(nil) + Jump bb4(v15, v16, v17) + bb4(v26:BasicObject, v27:BasicObject, v28:NilClass): + v34:ArrayExact = NewArray v27, v28 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_side_exit() { + eval(" + def test(a = (def foo = nil)) = a + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + v3:CPtr = LoadPC + v4:CPtr[CPtr(0x1000)] = Const CPtr(0x1008) + v5:CBool = IsBitEqual v3, v4 + IfTrue v5, bb2(v1, v2) + Jump bb4(v1, v2) + bb1(v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + Jump bb2(v9, v10) + bb2(v16:BasicObject, v17:BasicObject): + SideExit UnhandledYARVInsn(definemethod) + bb3(v13:BasicObject, v14:BasicObject): + EntryPoint JIT(1) + Jump bb4(v13, v14) + bb4(v22:BasicObject, v23:BasicObject): + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_setlocal_cyclic_default_args() { + eval(" + def test = proc { |a=a| a } + "); + assert_snapshot!(hir_string_proc("test"), @r" + fn block in test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb3(v9:BasicObject, v10:BasicObject): + EntryPoint JIT(1) + Jump bb2(v9, v10) + bb2(v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn defined_ivar() { + eval(" + def test = defined?(@foo) + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn if_defined_ivar() { + eval(" + def test + if defined?(@foo) + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:TrueClass|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + v13:CBool = Test v10 + IfFalse v13, bb3(v6) + v17:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v17 + bb3(v22:BasicObject): + v26:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn defined() { + eval(" + def test = return defined?(SeaChange), defined?(favourite), defined?($ruby) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:StringExact|NilClass = Defined constant, v10 + v15:StringExact|NilClass = Defined func, v6 + v17:NilClass = Const Value(nil) + v19:StringExact|NilClass = Defined global-variable, v17 + v21:ArrayExact = NewArray v12, v15, v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_return_const() { + eval(" + def test(cond) + if cond + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_leave); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :cond, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v15:CBool = Test v9 + IfFalse v15, bb3(v8, v9) + v19:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v19 + bb3(v24:BasicObject, v25:BasicObject): + v29:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_merge_const() { + eval(" + def test(cond) + if cond + result = 3 + else + result = 4 + end + result + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :cond, l0, SP@5 + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:NilClass): + CheckInterrupts + v18:CBool = Test v11 + IfFalse v18, bb3(v10, v11, v12) + v22:Fixnum[3] = Const Value(3) + CheckInterrupts + Jump bb4(v10, v11, v22) + bb3(v27:BasicObject, v28:BasicObject, v29:NilClass): + v33:Fixnum[4] = Const Value(4) + Jump bb4(v27, v28, v33) + bb4(v36:BasicObject, v37:BasicObject, v38:Fixnum): + CheckInterrupts + Return v38 + "); + } + + #[test] + fn test_opt_plus_fixnum() { + eval(" + def test(a, b) = a + b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :+, v12 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_minus_fixnum() { + eval(" + def test(a, b) = a - b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_minus); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :-, v12 # SendFallbackReason: Uncategorized(opt_minus) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_mult_fixnum() { + eval(" + def test(a, b) = a * b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mult); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :*, v12 # SendFallbackReason: Uncategorized(opt_mult) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_div_fixnum() { + eval(" + def test(a, b) = a / b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :/, v12 # SendFallbackReason: Uncategorized(opt_div) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_mod_fixnum() { + eval(" + def test(a, b) = a % b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :%, v12 # SendFallbackReason: Uncategorized(opt_mod) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_eq_fixnum() { + eval(" + def test(a, b) = a == b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :==, v12 # SendFallbackReason: Uncategorized(opt_eq) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_neq_fixnum() { + eval(" + def test(a, b) = a != b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :!=, v12 # SendFallbackReason: Uncategorized(opt_neq) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_lt_fixnum() { + eval(" + def test(a, b) = a < b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :<, v12 # SendFallbackReason: Uncategorized(opt_lt) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_le_fixnum() { + eval(" + def test(a, b) = a <= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_le); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :<=, v12 # SendFallbackReason: Uncategorized(opt_le) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_gt_fixnum() { + eval(" + def test(a, b) = a > b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_gt); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :>, v12 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_loop() { + eval(" + def test + result = 0 + times = 10 + while times > 0 + result = result + 1 + times = times - 1 + end + result + end + test + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject): + EntryPoint JIT(0) + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:Fixnum[0] = Const Value(0) + v20:Fixnum[10] = Const Value(10) + CheckInterrupts + Jump bb4(v10, v16, v20) + bb4(v26:BasicObject, v27:BasicObject, v28:BasicObject): + v32:Fixnum[0] = Const Value(0) + v35:BasicObject = SendWithoutBlock v28, :>, v32 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + v38:CBool = Test v35 + IfTrue v38, bb3(v26, v27, v28) + v41:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + bb3(v49:BasicObject, v50:BasicObject, v51:BasicObject): + v56:Fixnum[1] = Const Value(1) + v59:BasicObject = SendWithoutBlock v50, :+, v56 # SendFallbackReason: Uncategorized(opt_plus) + v64:Fixnum[1] = Const Value(1) + v67:BasicObject = SendWithoutBlock v51, :-, v64 # SendFallbackReason: Uncategorized(opt_minus) + Jump bb4(v49, v59, v67) + "); + } + + #[test] + fn test_opt_ge_fixnum() { + eval(" + def test(a, b) = a >= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_ge); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :>=, v12 # SendFallbackReason: Uncategorized(opt_ge) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_display_types() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb2(v1, v2) + bb1(v5:BasicObject): + EntryPoint JIT(0) + v6:NilClass = Const Value(nil) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v19:CBool[true] = Test v13 + IfFalse v19, bb3(v8, v13) + v23:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v23 + bb3(v28, v29): + v33 = Const Value(4) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_send_without_block() { + eval(" + def bar(a, b) + a+b + end + def test + bar(2, 3) + end + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + v13:Fixnum[3] = Const Value(3) + v15:BasicObject = SendWithoutBlock v6, :bar, v11, v13 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_send_with_block() { + eval(" + def test(a) + a.each {|item| + item + } + end + test([1,2,3]) + "); + assert_contains_opcode("test", YARVINSN_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:BasicObject = Send v9, 0x1000, :each # SendFallbackReason: Uncategorized(send) + v15:BasicObject = GetLocal :a, l0, EP@3 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_intern_interpolated_symbol() { + eval(r#" + def test + :"foo#{123}" + end + "#); + assert_contains_opcode("test", YARVINSN_intern); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[123] = Const Value(123) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + v21:Symbol = StringIntern v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn different_objects_get_addresses() { + eval("def test = unknown_method([0], [1], '2', '2')"); + + // The 2 string literals have the same address because they're deduped. + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:ArrayExact = ArrayDup v11 + v14:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v15:ArrayExact = ArrayDup v14 + v17:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v18:StringExact = StringCopy v17 + v20:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v21:StringExact = StringCopy v20 + v23:BasicObject = SendWithoutBlock v6, :unknown_method, v12, v15, v18, v21 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_cant_compile_splat() { + eval(" + def test(a) = foo(*a) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:ArrayExact = ToArray v9 + v17:BasicObject = SendWithoutBlock v8, :foo, v15 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_compile_block_arg() { + eval(" + def test(a) = foo(&a) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = Send v8, 0x1000, :foo, v9 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_cant_compile_kwarg() { + eval(" + def test(a) = foo(a: 1) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Fixnum[1] = Const Value(1) + v16:BasicObject = SendWithoutBlock v8, :foo, v14 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_cant_compile_kw_splat() { + eval(" + def test(a) = foo(**a) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v8, :foo, v9 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + // TODO(max): Figure out how to generate a call with TAILCALL flag + + #[test] + fn test_compile_super() { + eval(" + def test = super() + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_compile_zsuper() { + eval(" + def test = super + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_cant_compile_super_nil_blockarg() { + eval(" + def test = super(&nil) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:NilClass = Const Value(nil) + v13:BasicObject = InvokeSuper v6, 0x1000, v11 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_cant_compile_super_forward() { + eval(" + def test(...) = super(...) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + SideExit UnhandledYARVInsn(invokesuperforward) + "); + } + + #[test] + fn test_compile_forwardable() { + eval("def forwardable(...) = nil"); + assert_snapshot!(hir_string("forwardable"), @r" + fn forwardable@<compiled>:1: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:NilClass = Const Value(nil) + CheckInterrupts + Return v13 + "); + } + + // TODO(max): Figure out how to generate a call with OPT_SEND flag + + #[test] + fn test_cant_compile_kw_splat_mut() { + eval(" + def test(a) = foo **a, b: 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) + v16:HashExact = NewHash + PatchPoint NoEPEscape(test) + v21:BasicObject = SendWithoutBlock v14, :core#hash_merge_kwd, v16, v9 # SendFallbackReason: Uncategorized(opt_send_without_block) + v23:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) + v26:StaticSymbol[:b] = Const Value(VALUE(0x1008)) + v28:Fixnum[1] = Const Value(1) + v30:BasicObject = SendWithoutBlock v23, :core#hash_merge_ptr, v21, v26, v28 # SendFallbackReason: Uncategorized(opt_send_without_block) + v32:BasicObject = SendWithoutBlock v8, :foo, v30 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_cant_compile_splat_mut() { + eval(" + def test(*) = foo *, 1 + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:ArrayExact = GetLocal :*, l0, SP@4, * + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:ArrayExact): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:ArrayExact): + v15:ArrayExact = ToNewArray v9 + v17:Fixnum[1] = Const Value(1) + ArrayPush v15, v17 + v21:BasicObject = SendWithoutBlock v8, :foo, v15 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_compile_forwarding() { + eval(" + def test(...) = foo(...) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :..., l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendForward v8, 0x1000, :foo, v9 # SendFallbackReason: Uncategorized(sendforward) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_compile_triple_dots_with_positional_args() { + eval(" + def test(a, ...) = foo(a, ...) + "); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@8 + v3:ArrayExact = GetLocal :*, l0, SP@7, * + v4:BasicObject = GetLocal :**, l0, SP@6 + v5:BasicObject = GetLocal :&, l0, SP@5 + v6:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5, v6) + bb1(v9:BasicObject, v10:BasicObject, v11:ArrayExact, v12:BasicObject, v13:BasicObject): + EntryPoint JIT(0) + v14:NilClass = Const Value(nil) + Jump bb2(v9, v10, v11, v12, v13, v14) + bb2(v16:BasicObject, v17:BasicObject, v18:ArrayExact, v19:BasicObject, v20:BasicObject, v21:NilClass): + v28:ArrayExact = ToArray v18 + PatchPoint NoEPEscape(test) + GuardBlockParamProxy l0 + v34:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + SideExit UnhandledYARVInsn(splatkw) + "); + } + + #[test] + fn test_opt_new() { + eval(" + class C; end + def test = C.new + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v11:BasicObject = GetConstantPath 0x1000 + v13:NilClass = Const Value(nil) + v16:CBool = IsMethodCFunc v11, :new + IfFalse v16, bb3(v6, v13, v11) + v18:HeapBasicObject = ObjectAlloc v11 + v20:BasicObject = SendWithoutBlock v18, :initialize # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Jump bb4(v6, v18, v20) + bb3(v24:BasicObject, v25:NilClass, v26:BasicObject): + v29:BasicObject = SendWithoutBlock v26, :new # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb4(v24, v29, v25) + bb4(v32:BasicObject, v33:BasicObject, v34:BasicObject): + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_opt_newarray_send_max_no_elements() { + eval(" + def test = [].max + "); + // TODO(max): Rewrite to nil + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v11:BasicObject = ArrayMax + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_newarray_send_max() { + eval(" + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v19:BasicObject = ArrayMax v11, v12 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_opt_newarray_send_max_redefined() { + eval(" + class Array + alias_method :old_max, :max + def max + old_max * 2 + end + end + + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX)) + "); + } + + #[test] + fn test_opt_newarray_send_min() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].min + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + SideExit UnhandledNewarraySend(MIN) + "); + } + + #[test] + fn test_opt_newarray_send_hash() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH) + v32:Fixnum = ArrayHash v15, v16 + PatchPoint NoEPEscape(test) + v39:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v40:ArrayExact = ArrayDup v39 + v42:BasicObject = SendWithoutBlock v14, :puts, v40 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_opt_newarray_send_hash_redefined() { + eval(" + Array.class_eval { def hash = 42 } + + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:5: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH)) + "); + } + + #[test] + fn test_opt_newarray_send_pack() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].pack 'C' + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + v31:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v32:StringExact = StringCopy v31 + SideExit UnhandledNewarraySend(PACK) + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer() { + eval(r#" + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + v29:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v30:StringExact = StringCopy v29 + v36:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v37:StringExact = StringCopy v36 + v39:BasicObject = GetLocal :buf, l0, EP@3 + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK) + v42:String = ArrayPackBuffer v15, v16, fmt: v37, buf: v39 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer_redefined() { + eval(r#" + class Array + def pack(fmt, buffer: nil) = 5 + end + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:6: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + v29:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v30:StringExact = StringCopy v29 + v36:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v37:StringExact = StringCopy v36 + v39:BasicObject = GetLocal :buf, l0, EP@3 + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK)) + "); + } + + #[test] + fn test_opt_newarray_send_include_p() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v33:BoolExact = ArrayInclude v15, v16 | v16 + PatchPoint NoEPEscape(test) + v40:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v41:ArrayExact = ArrayDup v40 + v43:BasicObject = SendWithoutBlock v14, :puts, v41 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_opt_newarray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:10: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@7 + v3:BasicObject = GetLocal :b, l0, SP@6 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject): + EntryPoint JIT(0) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass): + v25:BasicObject = SendWithoutBlock v15, :+, v16 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_duparray_send_include_p() { + eval(" + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v15:BoolExact = DupArrayInclude VALUE(0x1000) | v9 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_opt_duparray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:9: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + v21:BasicObject = SendWithoutBlock v18, :length # SendFallbackReason: Uncategorized(opt_length) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:ArrayExact = NewArray v11, v12 + v21:BasicObject = SendWithoutBlock v18, :size # SendFallbackReason: Uncategorized(opt_size) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + test + "); + assert_contains_opcode("test", YARVINSN_getinstancevariable); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setinstancevariable); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_set_ivar_rescue_frozen() { + let result = eval(" + class Foo + attr_accessor :bar + def initialize + @bar = 1 + freeze + end + end + + def test(foo) + begin + foo.bar = 2 + rescue FrozenError + end + end + + foo = Foo.new + test(foo) + test(foo) + + foo.bar + "); + assert_eq!(VALUE::fixnum_from_usize(1), result); + } + + #[test] + fn test_getclassvariable() { + eval(" + class Foo + def self.test = @@foo + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_getclassvariable), "iseq Foo.test does not contain getclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = GetClassVar :@@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setclassvariable() { + eval(" + class Foo + def self.test = @@foo = 42 + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_setclassvariable), "iseq Foo.test does not contain setclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[42] = Const Value(42) + SetClassVar :@@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setglobal() { + eval(" + def test = $foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + SetGlobal :$foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getglobal() { + eval(" + def test = $foo + test + "); + assert_contains_opcode("test", YARVINSN_getglobal); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = GetGlobal :$foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_splatarray_mut() { + eval(" + def test(a) = [*a] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = ToNewArray v9 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_concattoarray() { + eval(" + def test(a) = [1, *a] + "); + assert_contains_opcode("test", YARVINSN_concattoarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:Fixnum[1] = Const Value(1) + v15:ArrayExact = NewArray v13 + v18:ArrayExact = ToArray v9 + ArrayExtend v15, v18 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_pushtoarray_one_element() { + eval(" + def test(a) = [*a, 1] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = ToNewArray v9 + v16:Fixnum[1] = Const Value(1) + ArrayPush v14, v16 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_pushtoarray_multiple_elements() { + eval(" + def test(a) = [*a, 1, 2, 3] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v14:ArrayExact = ToNewArray v9 + v16:Fixnum[1] = Const Value(1) + v18:Fixnum[2] = Const Value(2) + v20:Fixnum[3] = Const Value(3) + ArrayPush v14, v16 + ArrayPush v14, v18 + ArrayPush v14, v20 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_aset() { + eval(" + def test(a, b) = a[b] = 1 + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v16:NilClass = Const Value(nil) + v20:Fixnum[1] = Const Value(1) + v24:BasicObject = SendWithoutBlock v11, :[]=, v12, v20 # SendFallbackReason: Uncategorized(opt_aset) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_aref() { + eval(" + def test(a, b) = a[b] + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :a, l0, SP@5 + v3:BasicObject = GetLocal :b, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :[], v12 # SendFallbackReason: Uncategorized(opt_aref) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn opt_empty_p() { + eval(" + def test(x) = x.empty? + "); + assert_contains_opcode("test", YARVINSN_opt_empty_p); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :empty? # SendFallbackReason: Uncategorized(opt_empty_p) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn opt_succ() { + eval(" + def test(x) = x.succ + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :succ # SendFallbackReason: Uncategorized(opt_succ) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn opt_and() { + eval(" + def test(x, y) = x & y + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :&, v12 # SendFallbackReason: Uncategorized(opt_and) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn opt_or() { + eval(" + def test(x, y) = x | y + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :|, v12 # SendFallbackReason: Uncategorized(opt_or) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn opt_not() { + eval(" + def test(x) = !x + "); + assert_contains_opcode("test", YARVINSN_opt_not); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v15:BasicObject = SendWithoutBlock v9, :! # SendFallbackReason: Uncategorized(opt_not) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn opt_regexpmatch2() { + eval(" + def test(regexp, matchee) = regexp =~ matchee + "); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :regexp, l0, SP@5 + v3:BasicObject = GetLocal :matchee, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v19:BasicObject = SendWithoutBlock v11, :=~, v12 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + CheckInterrupts + Return v19 + "); + } + + #[test] + // Tests for ConstBase requires either constant or class definition, both + // of which can't be performed inside a method. + fn test_putspecialobject_vm_core_and_cbase() { + eval(" + def test + alias aliased __callee__ + end + "); + assert_contains_opcode("test", YARVINSN_putspecialobject); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) + v12:BasicObject = PutSpecialObject CBase + v14:StaticSymbol[:aliased] = Const Value(VALUE(0x1008)) + v16:StaticSymbol[:__callee__] = Const Value(VALUE(0x1010)) + v18:BasicObject = SendWithoutBlock v10, :core#set_method_alias, v12, v14, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn opt_reverse() { + eval(" + def reverse_odd + a, b, c = @a, @b, @c + [a, b, c] + end + + def reverse_even + a, b, c, d = @a, @b, @c, @d + [a, b, c, d] + end + "); + assert_contains_opcode("reverse_odd", YARVINSN_opt_reverse); + assert_contains_opcode("reverse_even", YARVINSN_opt_reverse); + assert_snapshot!(hir_strings!("reverse_odd", "reverse_even"), @r" + fn reverse_odd@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject): + EntryPoint JIT(0) + v8:NilClass = Const Value(nil) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:NilClass, v14:NilClass, v15:NilClass): + PatchPoint SingleRactorMode + v20:BasicObject = GetIvar v12, :@a + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v12, :@b + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v12, :@c + PatchPoint NoEPEscape(reverse_odd) + v38:ArrayExact = NewArray v20, v23, v26 + CheckInterrupts + Return v38 + + fn reverse_even@<compiled>:8: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject): + EntryPoint JIT(0) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:NilClass, v16:NilClass, v17:NilClass, v18:NilClass): + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v14, :@a + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v14, :@b + PatchPoint SingleRactorMode + v29:BasicObject = GetIvar v14, :@c + PatchPoint SingleRactorMode + v32:BasicObject = GetIvar v14, :@d + PatchPoint NoEPEscape(reverse_even) + v46:ArrayExact = NewArray v23, v26, v29, v32 + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_branchnil() { + eval(" + def test(x) = x&.itself + "); + assert_contains_opcode("test", YARVINSN_branchnil); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + CheckInterrupts + v16:CBool = IsNil v9 + IfTrue v16, bb3(v8, v9, v9) + v19:BasicObject = SendWithoutBlock v9, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb3(v8, v9, v19) + bb3(v21:BasicObject, v22:BasicObject, v23:BasicObject): + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_invokebuiltin_delegate_annotated() { + assert_contains_opcode("Float", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("Float"), @r" + fn Float@<internal:kernel>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :arg, l0, SP@6 + v3:BasicObject = GetLocal :exception, l0, SP@5 + v4:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject, v9:BasicObject): + EntryPoint JIT(0) + v10:Fixnum[0] = Const Value(0) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:BasicObject, v15:BasicObject): + v19:Float = InvokeBuiltin rb_f_float, v12, v13, v14 + Jump bb3(v12, v13, v14, v15, v19) + bb3(v21:BasicObject, v22:BasicObject, v23:BasicObject, v24:BasicObject, v25:Float): + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_invokebuiltin_cexpr_annotated() { + assert_contains_opcode("class", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("class"), @r" + fn class@<internal:kernel>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:HeapObject = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:HeapObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_delegate_with_args() { + // Using an unannotated builtin to test InvokeBuiltin generation + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Dir", "open")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate), "iseq Dir.open does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn open@<internal:dir>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :name, l0, SP@8 + v3:BasicObject = GetLocal :encoding, l0, SP@7 + v4:BasicObject = GetLocal <empty>, l0, SP@6 + v5:BasicObject = GetLocal :block, l0, SP@5 + v6:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5, v6) + bb1(v9:BasicObject, v10:BasicObject, v11:BasicObject, v13:BasicObject): + EntryPoint JIT(0) + v12:Fixnum[0] = Const Value(0) + v14:NilClass = Const Value(nil) + Jump bb2(v9, v10, v11, v12, v13, v14) + bb2(v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:NilClass): + v25:BasicObject = InvokeBuiltin dir_s_open, v16, v17, v18 + PatchPoint NoEPEscape(open) + GuardBlockParamProxy l0 + v32:HeapObject[BlockParamProxy] = Const Value(VALUE(0x1000)) + CheckInterrupts + v35:CBool[true] = Test v32 + IfFalse v35, bb3(v16, v17, v18, v19, v20, v25) + v40:BasicObject = InvokeBlock, v25 # SendFallbackReason: Uncategorized(invokeblock) + v43:BasicObject = InvokeBuiltin dir_s_close, v16, v25 + CheckInterrupts + Return v40 + bb3(v49, v50, v51, v52, v53, v54): + CheckInterrupts + Return v54 + "); + } + + #[test] + fn test_invokebuiltin_delegate_without_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "enable")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate_leave), "iseq GC.enable does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn enable@<internal:gc>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = InvokeBuiltin gc_enable, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_with_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "start")); + assert!(iseq_contains_opcode(iseq, YARVINSN_invokebuiltin), "iseq GC.start does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn start@<internal:gc>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :full_mark, l0, SP@7 + v3:BasicObject = GetLocal :immediate_mark, l0, SP@6 + v4:BasicObject = GetLocal :immediate_sweep, l0, SP@5 + v5:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject): + EntryPoint JIT(0) + v12:Fixnum[0] = Const Value(0) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:BasicObject): + v25:FalseClass = Const Value(false) + v27:BasicObject = InvokeBuiltin gc_start_internal, v14, v15, v16, v17, v25 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_name() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "name")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn name@<internal:symbol>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_to_s() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "to_s")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @r" + fn to_s@<internal:symbol>: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb3(v6, v10) + bb3(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dupn() { + eval(" + def test(x) = (x[0, 1] ||= 2) + "); + assert_contains_opcode("test", YARVINSN_dupn); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@4 + Jump bb2(v1, v2) + bb1(v5:BasicObject, v6:BasicObject): + EntryPoint JIT(0) + Jump bb2(v5, v6) + bb2(v8:BasicObject, v9:BasicObject): + v13:NilClass = Const Value(nil) + v16:Fixnum[0] = Const Value(0) + v18:Fixnum[1] = Const Value(1) + v21:BasicObject = SendWithoutBlock v9, :[], v16, v18 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + v25:CBool = Test v21 + IfTrue v25, bb3(v8, v9, v13, v9, v16, v18, v21) + v29:Fixnum[2] = Const Value(2) + v32:BasicObject = SendWithoutBlock v9, :[]=, v16, v18, v29 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v29 + bb3(v38:BasicObject, v39:BasicObject, v40:NilClass, v41:BasicObject, v42:Fixnum[0], v43:Fixnum[1], v44:BasicObject): + CheckInterrupts + Return v44 + "); + } + + #[test] + fn test_objtostring_anytostring() { + eval(" + def test = \"#{1}\" + "); + assert_contains_opcode("test", YARVINSN_objtostring); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_string_concat() { + eval(r##" + def test = "#{1}#{2}#{3}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:StringExact = StringConcat v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_string_concat_empty() { + eval(r##" + def test = "#{}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:NilClass = Const Value(nil) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_toregexp() { + eval(r##" + def test = /#{1}#{2}#{3}/ + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:RegexpExact = ToRegexp v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_toregexp_with_options() { + eval(r##" + def test = /#{1}#{2}/mixn + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:RegexpExact = ToRegexp v15, v22, MULTILINE|IGNORECASE|EXTENDED|NOENCODING + CheckInterrupts + Return v24 + "); + } + + #[test] + fn throw() { + eval(" + define_method(:throw_return) { return 1 } + define_method(:throw_break) { break 2 } + "); + assert_contains_opcode("throw_return", YARVINSN_throw); + assert_contains_opcode("throw_break", YARVINSN_throw); + assert_snapshot!(hir_strings!("throw_return", "throw_break"), @r" + fn block in <compiled>@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v12:Fixnum[1] = Const Value(1) + Throw TAG_RETURN, v12 + + fn block in <compiled>@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v12:Fixnum[2] = Const Value(2) + Throw TAG_BREAK, v12 + "); + } + + #[test] + fn test_invokeblock() { + eval(r#" + def test + yield + end + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb2(v1) + bb1(v4:BasicObject): + EntryPoint JIT(0) + Jump bb2(v4) + bb2(v6:BasicObject): + v10:BasicObject = InvokeBlock # SendFallbackReason: Uncategorized(invokeblock) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_invokeblock_with_args() { + eval(r#" + def test(x, y) + yield x, y + end + "#); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :x, l0, SP@5 + v3:BasicObject = GetLocal :y, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v18:BasicObject = InvokeBlock, v11, v12 # SendFallbackReason: Uncategorized(invokeblock) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_expandarray_no_splat() { + eval(r#" + def test(o) + a, b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@6 + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:NilClass, v15:NilClass): + v21:ArrayExact = GuardType v13, ArrayExact + v22:CInt64 = ArrayLength v21 + v23:CInt64[2] = GuardBitEquals v22, CInt64(2) + v24:CInt64[1] = Const CInt64(1) + v25:BasicObject = ArrayAref v21, v24 + v26:CInt64[0] = Const CInt64(0) + v27:BasicObject = ArrayAref v21, v26 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_expandarray_splat() { + eval(r#" + def test(o) + a, *b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@6 + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4) + bb1(v7:BasicObject, v8:BasicObject): + EntryPoint JIT(0) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb2(v7, v8, v9, v10) + bb2(v12:BasicObject, v13:BasicObject, v14:NilClass, v15:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_expandarray_splat_post() { + eval(r#" + def test(o) + a, *b, c = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:3: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :o, l0, SP@7 + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb2(v1, v2, v3, v4, v5) + bb1(v8:BasicObject, v9:BasicObject): + EntryPoint JIT(0) + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb2(v8, v9, v10, v11, v12) + bb2(v14:BasicObject, v15:BasicObject, v16:NilClass, v17:NilClass, v18:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_checkkeyword_tests_fixnum_bit() { + eval(r#" + def test(kw: 1 + 1) = kw + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :kw, l0, SP@5 + v3:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3) + bb1(v6:BasicObject, v7:BasicObject): + EntryPoint JIT(0) + v8:Fixnum[0] = Const Value(0) + Jump bb2(v6, v7, v8) + bb2(v10:BasicObject, v11:BasicObject, v12:BasicObject): + v15:BasicObject = GetLocal <empty>, l0, EP@3 + v16:BoolExact = FixnumBitCheck v15, 0 + CheckInterrupts + v19:CBool = Test v16 + IfTrue v19, bb3(v10, v11, v12) + v22:Fixnum[1] = Const Value(1) + v24:Fixnum[1] = Const Value(1) + v27:BasicObject = SendWithoutBlock v22, :+, v24 # SendFallbackReason: Uncategorized(opt_plus) + Jump bb3(v10, v27, v12) + bb3(v30:BasicObject, v31:BasicObject, v32:BasicObject): + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_checkkeyword_too_many_keywords_causes_side_exit() { + eval(r#" + def test(k1: k1, k2: k2, k3: k3, k4: k4, k5: k5, + k6: k6, k7: k7, k8: k8, k9: k9, k10: k10, k11: k11, + k12: k12, k13: k13, k14: k14, k15: k15, k16: k16, + k17: k17, k18: k18, k19: k19, k20: k20, k21: k21, + k22: k22, k23: k23, k24: k24, k25: k25, k26: k26, + k27: k27, k28: k28, k29: k29, k30: k30, k31: k31, + k32: k32, k33: k33) = k1 + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @r" + fn test@<compiled>:2: + bb0(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:BasicObject = GetLocal :k1, l0, SP@37 + v3:BasicObject = GetLocal :k2, l0, SP@36 + v4:BasicObject = GetLocal :k3, l0, SP@35 + v5:BasicObject = GetLocal :k4, l0, SP@34 + v6:BasicObject = GetLocal :k5, l0, SP@33 + v7:BasicObject = GetLocal :k6, l0, SP@32 + v8:BasicObject = GetLocal :k7, l0, SP@31 + v9:BasicObject = GetLocal :k8, l0, SP@30 + v10:BasicObject = GetLocal :k9, l0, SP@29 + v11:BasicObject = GetLocal :k10, l0, SP@28 + v12:BasicObject = GetLocal :k11, l0, SP@27 + v13:BasicObject = GetLocal :k12, l0, SP@26 + v14:BasicObject = GetLocal :k13, l0, SP@25 + v15:BasicObject = GetLocal :k14, l0, SP@24 + v16:BasicObject = GetLocal :k15, l0, SP@23 + v17:BasicObject = GetLocal :k16, l0, SP@22 + v18:BasicObject = GetLocal :k17, l0, SP@21 + v19:BasicObject = GetLocal :k18, l0, SP@20 + v20:BasicObject = GetLocal :k19, l0, SP@19 + v21:BasicObject = GetLocal :k20, l0, SP@18 + v22:BasicObject = GetLocal :k21, l0, SP@17 + v23:BasicObject = GetLocal :k22, l0, SP@16 + v24:BasicObject = GetLocal :k23, l0, SP@15 + v25:BasicObject = GetLocal :k24, l0, SP@14 + v26:BasicObject = GetLocal :k25, l0, SP@13 + v27:BasicObject = GetLocal :k26, l0, SP@12 + v28:BasicObject = GetLocal :k27, l0, SP@11 + v29:BasicObject = GetLocal :k28, l0, SP@10 + v30:BasicObject = GetLocal :k29, l0, SP@9 + v31:BasicObject = GetLocal :k30, l0, SP@8 + v32:BasicObject = GetLocal :k31, l0, SP@7 + v33:BasicObject = GetLocal :k32, l0, SP@6 + v34:BasicObject = GetLocal :k33, l0, SP@5 + v35:BasicObject = GetLocal <empty>, l0, SP@4 + Jump bb2(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35) + bb1(v38:BasicObject, v39:BasicObject, v40:BasicObject, v41:BasicObject, v42:BasicObject, v43:BasicObject, v44:BasicObject, v45:BasicObject, v46:BasicObject, v47:BasicObject, v48:BasicObject, v49:BasicObject, v50:BasicObject, v51:BasicObject, v52:BasicObject, v53:BasicObject, v54:BasicObject, v55:BasicObject, v56:BasicObject, v57:BasicObject, v58:BasicObject, v59:BasicObject, v60:BasicObject, v61:BasicObject, v62:BasicObject, v63:BasicObject, v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject, v68:BasicObject, v69:BasicObject, v70:BasicObject, v71:BasicObject): + EntryPoint JIT(0) + v72:Fixnum[0] = Const Value(0) + Jump bb2(v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63, v64, v65, v66, v67, v68, v69, v70, v71, v72) + bb2(v74:BasicObject, v75:BasicObject, v76:BasicObject, v77:BasicObject, v78:BasicObject, v79:BasicObject, v80:BasicObject, v81:BasicObject, v82:BasicObject, v83:BasicObject, v84:BasicObject, v85:BasicObject, v86:BasicObject, v87:BasicObject, v88:BasicObject, v89:BasicObject, v90:BasicObject, v91:BasicObject, v92:BasicObject, v93:BasicObject, v94:BasicObject, v95:BasicObject, v96:BasicObject, v97:BasicObject, v98:BasicObject, v99:BasicObject, v100:BasicObject, v101:BasicObject, v102:BasicObject, v103:BasicObject, v104:BasicObject, v105:BasicObject, v106:BasicObject, v107:BasicObject, v108:BasicObject): + SideExit TooManyKeywordParameters + "); + } + } + + /// Test successor and predecessor set computations. + #[cfg(test)] + mod control_flow_info_tests { + use super::*; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb1, bb2)); + assert!(cfi.is_succeeded_by(bb2, bb1)); + assert!(cfi.predecessors(bb3).eq([bb2])); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: v1, target: edge(bb2)}); + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb2, bb3)); + assert!(cfi.is_preceded_by(bb1, bb3)); + assert!(!cfi.is_preceded_by(bb0, bb3)); + assert!(cfi.is_succeeded_by(bb1, bb0)); + assert!(cfi.is_succeeded_by(bb3, bb1)); + } + + #[test] + fn test_cfi_deduplicated_successors_and_predecessors() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + // Construct two separate jump instructions. + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: v1, target: edge(bb1)}); + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + + assert_eq!(cfi.predecessors(bb1).collect::<Vec<_>>().len(), 1); + assert_eq!(cfi.successors(bb0).collect::<Vec<_>>().len(), 1); + } + } + + /// Test dominator set computations. + #[cfg(test)] + mod dom_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + fn assert_dominators_contains_self(function: &Function, dominators: &Dominators) { + for (i, _) in function.blocks.iter().enumerate() { + // Ensure that each dominating set contains the block itself. + assert!(dominators.is_dominated_by(BlockId(i), BlockId(i))); + } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + v3:Any = Const CBool(true) + Return v3 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb1, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb1, bb2, bb3].iter())); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val, target: edge(bb1)}); + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + IfTrue v0, bb1() + Jump bb2() + bb1(): + Jump bb3() + bb2(): + Jump bb3() + bb3(): + v5:Any = Const CBool(true) + Return v5 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb3].iter())); + } + + #[test] + fn test_complex_cfg() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + let bb7 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let v0 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::IfTrue { val: v0, target: edge(bb2)}); + function.push_insn(bb1, Insn::Jump(edge(bb4))); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let v1 = function.push_insn(bb3, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb3, Insn::IfTrue { val: v1, target: edge(bb5)}); + function.push_insn(bb3, Insn::Jump(edge(bb7))); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + function.push_insn(bb5, Insn::Jump(edge(bb6))); + + function.push_insn(bb6, Insn::Jump(edge(bb7))); + + let retval = function.push_insn(bb7, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb7, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + v1:Any = Const Value(false) + IfTrue v1, bb2() + Jump bb4() + bb2(): + Jump bb3() + bb3(): + v5:Any = Const Value(false) + IfTrue v5, bb5() + Jump bb7() + bb4(): + Jump bb5() + bb5(): + Jump bb6() + bb6(): + Jump bb7() + bb7(): + v11:Any = Const CBool(true) + Return v11 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb1, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb1, bb2, bb3].iter())); + assert!(dominators.dominators(bb4).eq([bb0, bb1, bb4].iter())); + assert!(dominators.dominators(bb5).eq([bb0, bb1, bb5].iter())); + assert!(dominators.dominators(bb6).eq([bb0, bb1, bb5, bb6].iter())); + assert!(dominators.dominators(bb7).eq([bb0, bb1, bb7].iter())); + } + + #[test] + fn test_back_edges() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + + let v0 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: v0, target: edge(bb1)}); + function.push_insn(bb0, Insn::Jump(edge(bb4))); + + let v1 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::IfTrue { val: v1, target: edge(bb2)}); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + let v2 = function.push_insn(bb5, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb5, Insn::IfTrue { val: v2, target: edge(bb3)}); + function.push_insn(bb5, Insn::Jump(edge(bb4))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + IfTrue v0, bb1() + Jump bb4() + bb1(): + v3:Any = Const Value(false) + IfTrue v3, bb2() + Jump bb3() + bb2(): + Jump bb3() + bb4(): + Jump bb5() + bb5(): + v8:Any = Const Value(false) + IfTrue v8, bb3() + Jump bb4() + bb3(): + v11:Any = Const CBool(true) + Return v11 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert!(dominators.dominators(bb0).eq([bb0].iter())); + assert!(dominators.dominators(bb1).eq([bb0, bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb0, bb1, bb2].iter())); + assert!(dominators.dominators(bb3).eq([bb0, bb3].iter())); + assert!(dominators.dominators(bb4).eq([bb0, bb4].iter())); + assert!(dominators.dominators(bb5).eq([bb0, bb4, bb5].iter())); + } + + #[test] + fn test_multiple_entry_blocks() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + function.jit_entry_blocks.push(bb1); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb2, Insn::Return { val: retval }); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb2() + bb1(): + Jump bb2() + bb2(): + v2:Any = Const CBool(true) + Return v2 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + + assert!(dominators.dominators(bb1).eq([bb1].iter())); + assert!(dominators.dominators(bb2).eq([bb2].iter())); + + assert!(!dominators.is_dominated_by(bb1, bb2)); + } + } + + /// Test loop information computation. +#[cfg(test)] +mod loop_info_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_loop_depth() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ ┌─────┐ + // │ bb2 ◄──────┼ bb1 ◄─┐ + // └──┬──┘ └─────┘ │ + // └─────────────────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::IfTrue { val, target: edge(bb1)}); + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb2() + v1:Any = Const Value(false) + bb2(): + IfTrue v1, bb1() + v3:Any = Const CBool(true) + Return v3 + bb1(): + Jump bb2() + "); + + assert!(loop_info.is_loop_header(bb2)); + assert!(loop_info.is_back_edge_source(bb1)); + assert_eq!(loop_info.loop_depth(bb1), 1); + } + + #[test] + fn test_nested_loops() { + // ┌─────┐ + // │ bb0 ◄─────┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄───┐ │ + // └──┬──┘ │ │ + // │ │ │ + // ┌──▼──┐ │ │ + // │ bb2 ┼───┘ │ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb3 ┼─────┘ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb4 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let cond = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let cond = function.push_insn(bb3, Insn::Const { val: Const::Value(Qtrue) }); + let _ = function.push_insn(bb3, Insn::IfTrue { val: cond, target: edge(bb0) }); + function.push_insn(bb3, Insn::Jump(edge(bb4))); + + let retval = function.push_insn(bb4, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb4, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + Jump bb2() + bb2(): + v2:Any = Const Value(false) + IfTrue v2, bb1() + Jump bb3() + bb3(): + v5:Any = Const Value(true) + IfTrue v5, bb0() + Jump bb4() + bb4(): + v8:Any = Const CBool(true) + Return v8 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 1); + assert_eq!(loop_info.loop_depth(bb4), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + } + + #[test] + fn test_complex_loops() { + // ┌─────┐ + // ┌──────► bb0 │ + // │ └──┬──┘ + // │ ┌────┴────┐ + // │ ┌──▼──┐ ┌──▼──┐ + // │ │ bb1 ◄─┐ │ bb3 ◄─┐ + // │ └──┬──┘ │ └──┬──┘ │ + // │ │ │ │ │ + // │ ┌──▼──┐ │ ┌──▼──┐ │ + // │ │ bb2 ┼─┘ │ bb4 ┼─┘ + // │ └──┬──┘ └──┬──┘ + // │ └────┬────┘ + // │ ┌──▼──┐ + // └──────┼ bb5 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb6 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb0, Insn::Jump(edge(bb3))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let _ = function.push_insn(bb2, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb2, Insn::Jump(edge(bb5))); + + function.push_insn(bb3, Insn::Jump(edge(bb4))); + + let _ = function.push_insn(bb4, Insn::IfTrue { val: cond, target: edge(bb3) }); + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + let _ = function.push_insn(bb5, Insn::IfTrue { val: cond, target: edge(bb0) }); + function.push_insn(bb5, Insn::Jump(edge(bb6))); + + let retval = function.push_insn(bb6, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb6, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + IfTrue v0, bb1() + Jump bb3() + bb1(): + Jump bb2() + bb2(): + IfTrue v0, bb1() + Jump bb5() + bb3(): + Jump bb4() + bb4(): + IfTrue v0, bb3() + Jump bb5() + bb5(): + IfTrue v0, bb0() + Jump bb6() + bb6(): + v11:Any = Const CBool(true) + Return v11 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + assert!(loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb5)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb6)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 2); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + assert_eq!(loop_info.loop_depth(bb6), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + } + + #[test] + fn linked_list_non_loop() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb1 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb2 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + Jump bb1() + bb1(): + Jump bb2() + bb2(): + v2:Any = Const CBool(true) + Return v2 + "); + + assert!(!loop_info.is_loop_header(bb0)); + assert!(!loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + + assert_eq!(loop_info.loop_depth(bb0), 0); + assert_eq!(loop_info.loop_depth(bb1), 0); + assert_eq!(loop_info.loop_depth(bb2), 0); + } + + #[test] + fn triple_nested_loop() { + // ┌─────┐ + // │ bb0 ◄──┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄─┐│ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb2 ◄┐││ + // └──┬──┘│││ + // │ │││ + // ┌──▼──┐│││ + // │ bb3 ┼┘││ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb4 ┼─┘│ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb5 ┼──┘ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + let _ = function.push_insn(bb2, Insn::Jump(edge(bb3))); + let _ = function.push_insn(bb3, Insn::Jump(edge(bb4))); + let _ = function.push_insn(bb3, Insn::IfTrue {val: cond, target: edge(bb2)}); + let _ = function.push_insn(bb4, Insn::Jump(edge(bb5))); + let _ = function.push_insn(bb4, Insn::IfTrue {val: cond, target: edge(bb1)}); + let _ = function.push_insn(bb5, Insn::IfTrue {val: cond, target: edge(bb0)}); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @r" + fn <manual>: + bb0(): + v0:Any = Const Value(false) + Jump bb1() + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + Jump bb4() + IfTrue v0, bb2() + bb4(): + Jump bb5() + IfTrue v0, bb1() + bb5(): + IfTrue v0, bb0() + "); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 3); + assert_eq!(loop_info.loop_depth(bb3), 3); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(loop_info.is_loop_header(bb2)); + assert!(!loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb5)); + } + } + +/// Test dumping to iongraph format. +#[cfg(test)] +mod iongraph_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_simple_function() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let retval = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: retval }); + + let json = function.to_iongraph_pass("simple"); + assert_snapshot!(json.to_string(), @r#"{"name":"simple", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_two_blocks() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + let json = function.to_iongraph_pass("two_blocks"); + assert_snapshot!(json.to_string(), @r#"{"name":"two_blocks", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb1()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4097, "id":1, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4098, "id":2, "opcode":"Return v1", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_instructions() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let val1 = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: val1 }); + + let json = function.to_iongraph_pass("multiple_instructions"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_instructions", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_conditional_branch() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::IfTrue { val: cond, target: edge(bb1) }); + + let retval1 = function.push_insn(bb0, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb0, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval2 }); + + let json = function.to_iongraph_pass("conditional_branch"); + assert_snapshot!(json.to_string(), @r#"{"name":"conditional_branch", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"IfTrue v0, bb1()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}, {"ptr":4098, "id":2, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4099, "id":3, "opcode":"Return v2", "attributes":[], "inputs":[2], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4100, "id":4, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4101, "id":5, "opcode":"Return v4", "attributes":[], "inputs":[4], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_loop_structure() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::IfTrue { val, target: edge(bb1)}); + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let json = function.to_iongraph_pass("loop_structure"); + assert_snapshot!(json.to_string(), @r#"{"name":"loop_structure", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[2], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}, {"ptr":4097, "id":1, "opcode":"Const Value(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}]}, {"ptr":4098, "id":2, "loopDepth":1, "attributes":["loopheader"], "predecessors":[0, 1], "successors":[1], "instructions":[{"ptr":4098, "id":2, "opcode":"IfTrue v1, bb1()", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}, {"ptr":4099, "id":3, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4100, "id":4, "opcode":"Return v3", "attributes":[], "inputs":[3], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":1, "attributes":["backedge"], "predecessors":[2], "successors":[2], "instructions":[{"ptr":4101, "id":5, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_successors() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::IfTrue { val: cond, target: edge(bb1) }); + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let retval1 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb2, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb2, Insn::Return { val: retval2 }); + + let json = function.to_iongraph_pass("multiple_successors"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_successors", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1, 2], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"IfTrue v0, bb1()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}, {"ptr":4098, "id":2, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4099, "id":3, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4100, "id":4, "opcode":"Return v3", "attributes":[], "inputs":[3], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4101, "id":5, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4102, "id":6, "opcode":"Return v5", "attributes":[], "inputs":[5], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + } diff --git a/zjit/src/hir_effect/gen_hir_effect.rb b/zjit/src/hir_effect/gen_hir_effect.rb new file mode 100644 index 0000000000..51cc712feb --- /dev/null +++ b/zjit/src/hir_effect/gen_hir_effect.rb @@ -0,0 +1,119 @@ +# Generate hir_effect.inc.rs. To do this, we build up a DAG that +# represents the ZJIT effect hierarchy. + +require 'set' + +# Effect represents not just a Ruby class but a named union of other effects. +class Effect + attr_accessor :name, :subeffects + + def initialize name, subeffects=nil + @name = name + @subeffects = subeffects || [] + end + + def all_subeffects + subeffects.flat_map { |subeffect| subeffect.all_subeffects } + subeffects + end + + def subeffect name + result = Effect.new name + @subeffects << result + result + end +end + +# Helper to generate graphviz. +def to_graphviz_rec effect + effect.subeffects.each {|subeffect| + puts effect.name + "->" + subeffect.name + ";" + } + effect.subeffect.each {|subeffect| + to_graphviz_rec subeffect + } +end + +# Generate graphviz. +def to_graphviz effect + puts "digraph G {" + to_graphviz_rec effect + puts "}" +end + +# ===== Start generating the effect DAG ===== + +# Start at Any. All effects are subeffects of Any. +any = Effect.new 'Any' +# Build the effect universe. +allocator = any.subeffect 'Allocator' +control = any.subeffect 'Control' +memory = any.subeffect 'Memory' +other = memory.subeffect 'Other' +frame = memory.subeffect 'Frame' +pc = frame.subeffect 'PC' +locals = frame.subeffect 'Locals' +stack = frame.subeffect 'Stack' + +# Use the smallest unsigned value needed to describe all effect bits +# If it becomes an issue, this can be generated but for now we do it manually +$int_label = 'u8' + +# Assign individual bits to effect leaves and union bit patterns to nodes with subeffects +num_bits = 0 +$bits = {"Empty" => ["0#{$int_label}"]} +$numeric_bits = {"Empty" => 0} +Set[any, *any.all_subeffects].sort_by(&:name).each {|effect| + subeffects = effect.subeffects + if subeffects.empty? + # Assign bits for leaves + $bits[effect.name] = ["1#{$int_label} << #{num_bits}"] + $numeric_bits[effect.name] = 1 << num_bits + num_bits += 1 + else + # Assign bits for unions + $bits[effect.name] = subeffects.map(&:name).sort + end +} +[*any.all_subeffects, any].each {|effect| + subeffects = effect.subeffects + unless subeffects.empty? + $numeric_bits[effect.name] = subeffects.map {|ty| $numeric_bits[ty.name]}.reduce(&:|) + end +} + +# ===== Finished generating the DAG; write Rust code ===== + +puts "// This file is @generated by src/hir/gen_hir_effect.rb." +puts "mod bits {" +$bits.keys.sort.map {|effect_name| + subeffects = $bits[effect_name].join(" | ") + puts " pub const #{effect_name}: #{$int_label} = #{subeffects};" +} +puts " pub const AllBitPatterns: [(&str, #{$int_label}); #{$bits.size}] = [" +# Sort the bit patterns by decreasing value so that we can print the densest +# possible to-string representation of an Effect. For example, Frame instead of +# PC|Stack|Locals +$numeric_bits.sort_by {|key, val| -val}.each {|effect_name, _| + puts " (\"#{effect_name}\", #{effect_name})," +} +puts " ];" +puts " pub const NumEffectBits: #{$int_label} = #{num_bits}; +}" + +puts "pub mod effect_types {" +puts " pub type EffectBits = #{$int_label};" +puts "}" + +puts "pub mod abstract_heaps { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: AbstractHeap = AbstractHeap::from_bits(bits::#{effect_name});" +} +puts "}" + +puts "pub mod effects { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: Effect = Effect::promote(abstract_heaps::#{effect_name});" +} +puts "}" diff --git a/zjit/src/hir_effect/hir_effect.inc.rs b/zjit/src/hir_effect/hir_effect.inc.rs new file mode 100644 index 0000000000..d9566b3eaa --- /dev/null +++ b/zjit/src/hir_effect/hir_effect.inc.rs @@ -0,0 +1,55 @@ +// This file is @generated by src/hir/gen_hir_effect.rb. +mod bits { + pub const Allocator: u8 = 1u8 << 0; + pub const Any: u8 = Allocator | Control | Memory; + pub const Control: u8 = 1u8 << 1; + pub const Empty: u8 = 0u8; + pub const Frame: u8 = Locals | PC | Stack; + pub const Locals: u8 = 1u8 << 2; + pub const Memory: u8 = Frame | Other; + pub const Other: u8 = 1u8 << 3; + pub const PC: u8 = 1u8 << 4; + pub const Stack: u8 = 1u8 << 5; + pub const AllBitPatterns: [(&str, u8); 10] = [ + ("Any", Any), + ("Memory", Memory), + ("Frame", Frame), + ("Stack", Stack), + ("PC", PC), + ("Other", Other), + ("Locals", Locals), + ("Control", Control), + ("Allocator", Allocator), + ("Empty", Empty), + ]; + pub const NumEffectBits: u8 = 6; +} +pub mod effect_types { + pub type EffectBits = u8; +} +pub mod abstract_heaps { + use super::*; + pub const Allocator: AbstractHeap = AbstractHeap::from_bits(bits::Allocator); + pub const Any: AbstractHeap = AbstractHeap::from_bits(bits::Any); + pub const Control: AbstractHeap = AbstractHeap::from_bits(bits::Control); + pub const Empty: AbstractHeap = AbstractHeap::from_bits(bits::Empty); + pub const Frame: AbstractHeap = AbstractHeap::from_bits(bits::Frame); + pub const Locals: AbstractHeap = AbstractHeap::from_bits(bits::Locals); + pub const Memory: AbstractHeap = AbstractHeap::from_bits(bits::Memory); + pub const Other: AbstractHeap = AbstractHeap::from_bits(bits::Other); + pub const PC: AbstractHeap = AbstractHeap::from_bits(bits::PC); + pub const Stack: AbstractHeap = AbstractHeap::from_bits(bits::Stack); +} +pub mod effects { + use super::*; + pub const Allocator: Effect = Effect::promote(abstract_heaps::Allocator); + pub const Any: Effect = Effect::promote(abstract_heaps::Any); + pub const Control: Effect = Effect::promote(abstract_heaps::Control); + pub const Empty: Effect = Effect::promote(abstract_heaps::Empty); + pub const Frame: Effect = Effect::promote(abstract_heaps::Frame); + pub const Locals: Effect = Effect::promote(abstract_heaps::Locals); + pub const Memory: Effect = Effect::promote(abstract_heaps::Memory); + pub const Other: Effect = Effect::promote(abstract_heaps::Other); + pub const PC: Effect = Effect::promote(abstract_heaps::PC); + pub const Stack: Effect = Effect::promote(abstract_heaps::Stack); +} diff --git a/zjit/src/hir_effect/mod.rs b/zjit/src/hir_effect/mod.rs new file mode 100644 index 0000000000..b1d7b27411 --- /dev/null +++ b/zjit/src/hir_effect/mod.rs @@ -0,0 +1,420 @@ +//! High-level intermediate representation effects. + +#![allow(non_upper_case_globals)] +use crate::hir::{PtrPrintMap}; +include!("hir_effect.inc.rs"); + +// NOTE: Effect very intentionally does not support Eq or PartialEq; we almost never want to check +// bit equality of types in the compiler but instead check subtyping, intersection, union, etc. +/// The AbstractHeap struct is the main work horse of effect inference and specialization. The main interfaces +/// will look like: +/// +/// * is AbstractHeap A a subset of AbstractHeap B +/// * union/meet AbstractHeap A and AbstractHeap B +/// +/// or +/// +/// * is Effect A a subset of Effect B +/// * union/meet Effect A and Effect B +/// +/// The AbstractHeap is the work horse because Effect is simply 2 AbstractHeaps; one for read, and one for write. +/// Currently, the abstract heap is implemented as a bitset. As we enrich our effect system, this will be updated +/// to match the name and use a heap implementation, roughly aligned with +/// <https://gist.github.com/pizlonator/cf1e72b8600b1437dda8153ea3fdb963>. +/// +/// Most questions can be rewritten in terms of these operations. +/// +/// Lattice Top corresponds to the "Any" effect. All bits are set and any effect is possible. +/// Lattice Bottom corresponds to the "None" effect. No bits are set and no effects are possible. +/// Elements between abstract_heaps have effects corresponding to the bits that are set. +/// This enables more complex analyses compared to prior ZJIT implementations such as "has_effect", +/// a function that returns a boolean value. Such functions impose an implicit single bit effect +/// system. This explicit design with a lattice allows us many bits for effects. +#[derive(Clone, Copy, Debug)] +pub struct AbstractHeap { + bits: effect_types::EffectBits +} + +#[derive(Clone, Copy, Debug)] +pub struct Effect { + /// Unlike ZJIT's type system, effects do not have a notion of subclasses. + /// Instead of specializations, the Effect struct contains two AbstractHeaps. + /// We distinguish between read effects and write effects. + /// Both use the same effects lattice, but splitting into two heaps allows + /// for finer grained optimization. + /// + /// For instance: + /// We can elide HIR instructions with no write effects, but the read effects are necessary for instruction + /// reordering optimizations. + /// + /// These fields should not be directly read or written except by internal `Effect` APIs. + read: AbstractHeap, + write: AbstractHeap +} + +/// Print adaptor for [`AbstractHeap`]. See [`PtrPrintMap`]. +pub struct AbstractHeapPrinter<'a> { + inner: AbstractHeap, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for AbstractHeapPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let effect = self.inner; + let mut bits = effect.bits; + let mut sep = ""; + // First, make sure patterns are sorted from higher order bits to lower order. + // For each match where `bits` contains the pattern, we mask off the matched bits + // and continue searching for matches until bits == 0. + // Our first match could be exact and may not require a separator, but all subsequent + // matches do. + debug_assert!(bits::AllBitPatterns.is_sorted_by(|(_, left), (_, right)| left > right)); + for (name, pattern) in bits::AllBitPatterns { + if (bits & pattern) == pattern { + write!(f, "{sep}{name}")?; + sep = "|"; + bits &= !pattern; + } + // The `sep != ""` check allows us to handle the effects::None case gracefully. + if (bits == 0) & (sep != "") { break; } + } + debug_assert_eq!(bits, 0, "Should have eliminated all bits by iterating over all patterns"); + Ok(()) + } +} + +impl std::fmt::Display for AbstractHeap { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +/// Print adaptor for [`Effect`]. See [`PtrPrintMap`]. +pub struct EffectPrinter<'a> { + inner: Effect, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for EffectPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}, {}", self.inner.read, self.inner.write) + } +} + +impl std::fmt::Display for Effect { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl AbstractHeap { + const fn from_bits(bits: effect_types::EffectBits) -> Self { + Self { bits } + } + + pub const fn union(&self, other: Self) -> Self { + Self::from_bits(self.bits | other.bits) + } + + pub const fn intersect(&self, other: Self) -> Self { + Self::from_bits(self.bits & other.bits) + } + + pub const fn exclude(&self, other: Self) -> Self { + Self::from_bits(self.bits - (self.bits & other.bits)) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Self) -> bool { + self.bits == other.bits + } + + pub const fn includes(&self, other: Self) -> bool { + self.bit_equal( + self.union(other) + ) + } + + pub const fn overlaps(&self, other: Self) -> bool { + !abstract_heaps::Empty.includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> AbstractHeapPrinter<'_> { + AbstractHeapPrinter { inner: self, ptr_map } + } +} + +impl Effect { + pub const fn read_write(read: AbstractHeap, write: AbstractHeap) -> Effect { + Effect { read, write } + } + + /// This function addresses the special case where the read and write heaps are the same + pub const fn promote(heap: AbstractHeap) -> Effect { + Effect {read: heap, write: heap } + } + + /// This function accepts write and heaps read to Any + pub const fn write(write: AbstractHeap) -> Effect { + Effect { read: abstract_heaps::Any, write } + } + + /// This function accepts read and heaps read to Any + pub const fn read(read: AbstractHeap) -> Effect { + Effect { read, write: abstract_heaps::Any } + } + + /// Method to access the private read field + pub const fn read_bits(&self) -> AbstractHeap { + self.read + } + + /// Method to access the private write field + pub const fn write_bits(&self) -> AbstractHeap { + self.write + } + + pub const fn union(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.union(other.read), + self.write.union(other.write) + ) + } + + pub const fn intersect(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.intersect(other.read), + self.write.intersect(other.write) + ) + } + + pub const fn exclude(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.exclude(other.read), + self.write.exclude(other.write) + ) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Effect) -> bool { + self.read.bit_equal(other.read) & self.write.bit_equal(other.write) + } + + pub const fn includes(&self, other: Effect) -> bool { + self.bit_equal(Effect::union(self, other)) + } + + pub const fn overlaps(&self, other: Effect) -> bool { + Effect::promote(abstract_heaps::Empty).includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> EffectPrinter<'_> { + EffectPrinter { inner: self, ptr_map } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[track_caller] + fn assert_heap_bit_equal(left: AbstractHeap, right: AbstractHeap) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(right.includes(left), "{left} is not a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_not_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(!right.includes(left), "{left} is a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_bit_equal(left: Effect, right: Effect) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect(left: Effect, right: Effect) { + assert!(right.includes(left), "{left} is not a subeffect of {right}"); + } + + #[track_caller] + fn assert_not_subeffect(left: Effect, right: Effect) { + assert!(!right.includes(left), "{left} is a subeffect of {right}"); + } + + #[test] + fn effect_heap_none_is_subeffect_of_everything() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Control); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Stack); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Locals); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Allocator); + } + + #[test] + fn effect_heap_everything_is_subeffect_of_any() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Control, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Any); + } + + #[test] + fn effect_heap_union_never_shrinks() { + // iterate over all effect entries from bottom to top + for i in [0, 1, 4, 6, 10, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty.union(e)); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any.union(e)); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Frame.union(e)); + } + } + + #[test] + fn effect_heap_intersect_never_grows() { + // Randomly selected values from bottom to top + for i in [0, 3, 6, 8, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty.intersect(e), abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Any.intersect(e), abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame.intersect(e), abstract_heaps::Frame); + } + } + + #[test] + fn effect_heap_self_is_included() { + assert!(abstract_heaps::Stack.includes(abstract_heaps::Stack)); + assert!(abstract_heaps::Any.includes(abstract_heaps::Any)); + assert!(abstract_heaps::Empty.includes(abstract_heaps::Empty)); + } + + #[test] + fn effect_heap_frame_includes_stack_locals_and_pc() { + assert_subeffect_heap(abstract_heaps::Stack, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Frame); + } + + #[test] + fn effect_heap_frame_is_stack_locals_and_pc() { + let union = abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC)); + assert_heap_bit_equal(abstract_heaps::Frame, union); + } + + #[test] + fn effect_heap_any_includes_some_subeffects() { + assert_subeffect_heap(abstract_heaps::Allocator, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + } + + #[test] + fn effect_heap_display_exact_bits_match() { + assert_eq!(format!("{}", abstract_heaps::Empty), "Empty"); + assert_eq!(format!("{}", abstract_heaps::PC), "PC"); + assert_eq!(format!("{}", abstract_heaps::Any), "Any"); + assert_eq!(format!("{}", abstract_heaps::Frame), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + } + + #[test] + fn effect_heap_display_multiple_bits() { + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals)), "Stack|Locals"); + assert_eq!(format!("{}", abstract_heaps::PC.union(abstract_heaps::Allocator)), "PC|Allocator"); + } + + #[test] + fn effect_any_includes_everything() { + assert_subeffect(effects::Allocator, effects::Any); + assert_subeffect(effects::Frame, effects::Any); + assert_subeffect(effects::Memory, effects::Any); + // Let's do a less standard effect too + assert_subeffect( + Effect::read_write(abstract_heaps::Control, abstract_heaps::Any), + effects::Any + ); + } + + #[test] + fn effect_union_is_idempotent() { + assert_bit_equal( + Effect::read(abstract_heaps::Any) + .union(Effect::write(abstract_heaps::Any)), + effects::Any + ); + assert_bit_equal( + effects::Empty.union(effects::Empty), + effects::Empty + ); + } + + #[test] + fn effect_union_contains_and_excludes() { + assert_subeffect( + effects::Control.union(effects::Frame), + effects::Any + ); + assert_not_subeffect( + effects::Frame.union(effects::Locals), + effects::PC + ); + } + + #[test] + fn effect_intersect_is_empty() { + assert_subeffect(effects::Memory.intersect(effects::Control), effects::Empty); + assert_subeffect( + Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Other) + .intersect(Effect::read_write(abstract_heaps::Stack, abstract_heaps::PC)), + effects::Empty + ) + } + + #[test] + fn effect_intersect_exact_match() { + assert_subeffect(effects::Frame.intersect(effects::PC), effects::PC); + assert_subeffect(effects::Allocator.intersect(effects::Allocator), effects::Allocator); + } + + #[test] + fn effect_display_exact_bits_match() { + assert_eq!(format!("{}", effects::Empty), "Empty, Empty"); + assert_eq!(format!("{}", effects::PC), "PC, PC"); + assert_eq!(format!("{}", effects::Any), "Any, Any"); + assert_eq!(format!("{}", effects::Frame), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", Effect::write(abstract_heaps::Control)), "Any, Control"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Memory)), "Allocator, Memory"); + } + + #[test] + fn effect_display_multiple_bits() { + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals)), "Stack|Locals, Stack|Locals"); + assert_eq!(format!("{}", effects::PC.union(effects::Allocator)), "PC|Allocator, PC|Allocator"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Other, abstract_heaps::PC) + .union(Effect::read_write(abstract_heaps::Memory, abstract_heaps::Stack))), + "Memory, Stack|PC" + ); + } + +} diff --git a/zjit/src/hir_type/gen_hir_type.rb b/zjit/src/hir_type/gen_hir_type.rb index ad227ef7b8..9576d2b1c0 100644 --- a/zjit/src/hir_type/gen_hir_type.rb +++ b/zjit/src/hir_type/gen_hir_type.rb @@ -25,20 +25,20 @@ class Type end # Helper to generate graphviz. -def to_graphviz_rec type +def to_graphviz_rec type, f type.subtypes.each {|subtype| - puts type.name + "->" + subtype.name + ";" + f.puts type.name + "->" + subtype.name + ";" } type.subtypes.each {|subtype| - to_graphviz_rec subtype + to_graphviz_rec subtype, f } end # Generate graphviz. -def to_graphviz type - puts "digraph G {" - to_graphviz_rec type - puts "}" +def to_graphviz type, f + f.puts "digraph G {" + to_graphviz_rec type, f + f.puts "}" end # ===== Start generating the type DAG ===== @@ -48,6 +48,7 @@ any = Type.new "Any" # Build the Ruby object universe. value = any.subtype "RubyValue" undef_ = value.subtype "Undef" +value.subtype "CallableMethodEntry" # rb_callable_method_entry_t* basic_object = value.subtype "BasicObject" basic_object_exact = basic_object.subtype "BasicObjectExact" basic_object_subclass = basic_object.subtype "BasicObjectSubclass" @@ -57,38 +58,71 @@ object_subclass = $object.subtype "ObjectSubclass" $subclass = [basic_object_subclass.name, object_subclass.name] $builtin_exact = [basic_object_exact.name, object_exact.name] +$exact_c_names = { + "ObjectExact" => "rb_cObject", + "BasicObjectExact" => "rb_cBasicObject", +} + +$inexact_c_names = { + "Object" => "rb_cObject", + "BasicObject" => "rb_cBasicObject", +} + # Define a new type that can be subclassed (most of them). -def base_type name +# If c_name is given, mark the rb_cXYZ object as equivalent to this exact type. +def base_type name, c_name: nil type = $object.subtype name exact = type.subtype(name+"Exact") subclass = type.subtype(name+"Subclass") + if c_name + $exact_c_names[exact.name] = c_name + $inexact_c_names[subclass.name] = c_name + end $builtin_exact << exact.name $subclass << subclass.name [type, exact] end -base_type "String" -base_type "Array" -base_type "Hash" +# Define a new type that cannot be subclassed. +# If c_name is given, mark the rb_cXYZ object as equivalent to this type. +def final_type name, base: $object, c_name: nil + if c_name + $exact_c_names[name] = c_name + end + type = base.subtype name + $builtin_exact << type.name + type +end + +base_type "String", c_name: "rb_cString" +base_type "Array", c_name: "rb_cArray" +base_type "Hash", c_name: "rb_cHash" +base_type "Range", c_name: "rb_cRange" +base_type "Set", c_name: "rb_cSet" +base_type "Regexp", c_name: "rb_cRegexp" +module_class, _ = base_type "Module", c_name: "rb_cModule" +class_ = final_type "Class", base: module_class, c_name: "rb_cClass" -(integer, integer_exact) = base_type "Integer" +numeric, _ = base_type "Numeric", c_name: "rb_cNumeric" + +integer_exact = final_type "Integer", base: numeric, c_name: "rb_cInteger" # CRuby partitions Integer into immediate and non-immediate variants. fixnum = integer_exact.subtype "Fixnum" integer_exact.subtype "Bignum" -(float, float_exact) = base_type "Float" +float_exact = final_type "Float", base: numeric, c_name: "rb_cFloat" # CRuby partitions Float into immediate and non-immediate variants. flonum = float_exact.subtype "Flonum" float_exact.subtype "HeapFloat" -(symbol, symbol_exact) = base_type "Symbol" +symbol_exact = final_type "Symbol", c_name: "rb_cSymbol" # CRuby partitions Symbol into immediate and non-immediate variants. static_sym = symbol_exact.subtype "StaticSymbol" symbol_exact.subtype "DynamicSymbol" -_, nil_exact = base_type "NilClass" -_, true_exact = base_type "TrueClass" -_, false_exact = base_type "FalseClass" +nil_exact = final_type "NilClass", c_name: "rb_cNilClass" +true_exact = final_type "TrueClass", c_name: "rb_cTrueClass" +false_exact = final_type "FalseClass", c_name: "rb_cFalseClass" # Build the cvalue object universe. This is for C-level types that may be # passed around when calling into the Ruby VM or after some strength reduction @@ -105,6 +139,7 @@ unsigned = cvalue_int.subtype "CUnsigned" signed.subtype "CInt#{width}" unsigned.subtype "CUInt#{width}" } +unsigned.subtype "CShape" # Assign individual bits to type leaves and union bit patterns to nodes with subtypes num_bits = 0 @@ -143,6 +178,10 @@ add_union "BuiltinExact", $builtin_exact add_union "Subclass", $subclass add_union "BoolExact", [true_exact.name, false_exact.name] add_union "Immediate", [fixnum.name, flonum.name, static_sym.name, nil_exact.name, true_exact.name, false_exact.name, undef_.name] +$bits["HeapBasicObject"] = ["BasicObject & !Immediate"] +$numeric_bits["HeapBasicObject"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Immediate"] +$bits["HeapObject"] = ["Object & !Immediate"] +$numeric_bits["HeapObject"] = $numeric_bits["Object"] & ~$numeric_bits["Immediate"] # ===== Finished generating the DAG; write Rust code ===== @@ -152,7 +191,7 @@ $bits.keys.sort.map {|type_name| subtypes = $bits[type_name].join(" | ") puts " pub const #{type_name}: u64 = #{subtypes};" } -puts " pub const AllBitPatterns: [(&'static str, u64); #{$bits.size}] = [" +puts " pub const AllBitPatterns: [(&str, u64); #{$bits.size}] = [" # Sort the bit patterns by decreasing value so that we can print the densest # possible to-string representation of a Type. For example, CSigned instead of # CInt8|CInt16|... @@ -168,4 +207,19 @@ puts "pub mod types { $bits.keys.sort.map {|type_name| puts " pub const #{type_name}: Type = Type::from_bits(bits::#{type_name});" } +puts " pub const ExactBitsAndClass: [(u64, *const VALUE); #{$exact_c_names.size}] = [" +$exact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +$inexact_c_names = $inexact_c_names.to_a.sort_by {|name, _| $bits[name]}.to_h +puts " pub const InexactBitsAndClass: [(u64, *const VALUE); #{$inexact_c_names.size}] = [" +$inexact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" puts "}" + +File.open("zjit_types.dot", "w") do |f| + to_graphviz(any, f) +end diff --git a/zjit/src/hir_type/hir_type.inc.rs b/zjit/src/hir_type/hir_type.inc.rs index e4717efadf..b388b3a0d1 100644 --- a/zjit/src/hir_type/hir_type.inc.rs +++ b/zjit/src/hir_type/hir_type.inc.rs @@ -8,8 +8,8 @@ mod bits { pub const BasicObjectExact: u64 = 1u64 << 2; pub const BasicObjectSubclass: u64 = 1u64 << 3; pub const Bignum: u64 = 1u64 << 4; - pub const BoolExact: u64 = FalseClassExact | TrueClassExact; - pub const BuiltinExact: u64 = ArrayExact | BasicObjectExact | FalseClassExact | FloatExact | HashExact | IntegerExact | NilClassExact | ObjectExact | StringExact | SymbolExact | TrueClassExact; + pub const BoolExact: u64 = FalseClass | TrueClass; + pub const BuiltinExact: u64 = ArrayExact | BasicObjectExact | Class | FalseClass | Float | HashExact | Integer | ModuleExact | NilClass | NumericExact | ObjectExact | RangeExact | RegexpExact | SetExact | StringExact | Symbol | TrueClass; pub const CBool: u64 = 1u64 << 5; pub const CDouble: u64 = 1u64 << 6; pub const CInt: u64 = CSigned | CUnsigned; @@ -19,91 +19,106 @@ mod bits { pub const CInt8: u64 = 1u64 << 10; pub const CNull: u64 = 1u64 << 11; pub const CPtr: u64 = 1u64 << 12; + pub const CShape: u64 = 1u64 << 13; pub const CSigned: u64 = CInt16 | CInt32 | CInt64 | CInt8; - pub const CUInt16: u64 = 1u64 << 13; - pub const CUInt32: u64 = 1u64 << 14; - pub const CUInt64: u64 = 1u64 << 15; - pub const CUInt8: u64 = 1u64 << 16; - pub const CUnsigned: u64 = CUInt16 | CUInt32 | CUInt64 | CUInt8; + pub const CUInt16: u64 = 1u64 << 14; + pub const CUInt32: u64 = 1u64 << 15; + pub const CUInt64: u64 = 1u64 << 16; + pub const CUInt8: u64 = 1u64 << 17; + pub const CUnsigned: u64 = CShape | CUInt16 | CUInt32 | CUInt64 | CUInt8; pub const CValue: u64 = CBool | CDouble | CInt | CNull | CPtr; - pub const DynamicSymbol: u64 = 1u64 << 17; + pub const CallableMethodEntry: u64 = 1u64 << 18; + pub const Class: u64 = 1u64 << 19; + pub const DynamicSymbol: u64 = 1u64 << 20; pub const Empty: u64 = 0u64; - pub const FalseClass: u64 = FalseClassExact | FalseClassSubclass; - pub const FalseClassExact: u64 = 1u64 << 18; - pub const FalseClassSubclass: u64 = 1u64 << 19; - pub const Fixnum: u64 = 1u64 << 20; - pub const Float: u64 = FloatExact | FloatSubclass; - pub const FloatExact: u64 = Flonum | HeapFloat; - pub const FloatSubclass: u64 = 1u64 << 21; - pub const Flonum: u64 = 1u64 << 22; + pub const FalseClass: u64 = 1u64 << 21; + pub const Fixnum: u64 = 1u64 << 22; + pub const Float: u64 = Flonum | HeapFloat; + pub const Flonum: u64 = 1u64 << 23; pub const Hash: u64 = HashExact | HashSubclass; - pub const HashExact: u64 = 1u64 << 23; - pub const HashSubclass: u64 = 1u64 << 24; - pub const HeapFloat: u64 = 1u64 << 25; - pub const Immediate: u64 = FalseClassExact | Fixnum | Flonum | NilClassExact | StaticSymbol | TrueClassExact | Undef; - pub const Integer: u64 = IntegerExact | IntegerSubclass; - pub const IntegerExact: u64 = Bignum | Fixnum; - pub const IntegerSubclass: u64 = 1u64 << 26; - pub const NilClass: u64 = NilClassExact | NilClassSubclass; - pub const NilClassExact: u64 = 1u64 << 27; - pub const NilClassSubclass: u64 = 1u64 << 28; - pub const Object: u64 = Array | FalseClass | Float | Hash | Integer | NilClass | ObjectExact | ObjectSubclass | String | Symbol | TrueClass; - pub const ObjectExact: u64 = 1u64 << 29; - pub const ObjectSubclass: u64 = 1u64 << 30; - pub const RubyValue: u64 = BasicObject | Undef; - pub const StaticSymbol: u64 = 1u64 << 31; + pub const HashExact: u64 = 1u64 << 24; + pub const HashSubclass: u64 = 1u64 << 25; + pub const HeapBasicObject: u64 = BasicObject & !Immediate; + pub const HeapFloat: u64 = 1u64 << 26; + pub const HeapObject: u64 = Object & !Immediate; + pub const Immediate: u64 = FalseClass | Fixnum | Flonum | NilClass | StaticSymbol | TrueClass | Undef; + pub const Integer: u64 = Bignum | Fixnum; + pub const Module: u64 = Class | ModuleExact | ModuleSubclass; + pub const ModuleExact: u64 = 1u64 << 27; + pub const ModuleSubclass: u64 = 1u64 << 28; + pub const NilClass: u64 = 1u64 << 29; + pub const Numeric: u64 = Float | Integer | NumericExact | NumericSubclass; + pub const NumericExact: u64 = 1u64 << 30; + pub const NumericSubclass: u64 = 1u64 << 31; + pub const Object: u64 = Array | FalseClass | Hash | Module | NilClass | Numeric | ObjectExact | ObjectSubclass | Range | Regexp | Set | String | Symbol | TrueClass; + pub const ObjectExact: u64 = 1u64 << 32; + pub const ObjectSubclass: u64 = 1u64 << 33; + pub const Range: u64 = RangeExact | RangeSubclass; + pub const RangeExact: u64 = 1u64 << 34; + pub const RangeSubclass: u64 = 1u64 << 35; + pub const Regexp: u64 = RegexpExact | RegexpSubclass; + pub const RegexpExact: u64 = 1u64 << 36; + pub const RegexpSubclass: u64 = 1u64 << 37; + pub const RubyValue: u64 = BasicObject | CallableMethodEntry | Undef; + pub const Set: u64 = SetExact | SetSubclass; + pub const SetExact: u64 = 1u64 << 38; + pub const SetSubclass: u64 = 1u64 << 39; + pub const StaticSymbol: u64 = 1u64 << 40; pub const String: u64 = StringExact | StringSubclass; - pub const StringExact: u64 = 1u64 << 32; - pub const StringSubclass: u64 = 1u64 << 33; - pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | FalseClassSubclass | FloatSubclass | HashSubclass | IntegerSubclass | NilClassSubclass | ObjectSubclass | StringSubclass | SymbolSubclass | TrueClassSubclass; - pub const Symbol: u64 = SymbolExact | SymbolSubclass; - pub const SymbolExact: u64 = DynamicSymbol | StaticSymbol; - pub const SymbolSubclass: u64 = 1u64 << 34; - pub const TrueClass: u64 = TrueClassExact | TrueClassSubclass; - pub const TrueClassExact: u64 = 1u64 << 35; - pub const TrueClassSubclass: u64 = 1u64 << 36; - pub const Undef: u64 = 1u64 << 37; - pub const AllBitPatterns: [(&'static str, u64); 63] = [ + pub const StringExact: u64 = 1u64 << 41; + pub const StringSubclass: u64 = 1u64 << 42; + pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | HashSubclass | ModuleSubclass | NumericSubclass | ObjectSubclass | RangeSubclass | RegexpSubclass | SetSubclass | StringSubclass; + pub const Symbol: u64 = DynamicSymbol | StaticSymbol; + pub const TrueClass: u64 = 1u64 << 43; + pub const Undef: u64 = 1u64 << 44; + pub const AllBitPatterns: [(&str, u64); 71] = [ ("Any", Any), ("RubyValue", RubyValue), ("Immediate", Immediate), ("Undef", Undef), ("BasicObject", BasicObject), ("Object", Object), - ("TrueClass", TrueClass), - ("Subclass", Subclass), - ("TrueClassSubclass", TrueClassSubclass), ("BuiltinExact", BuiltinExact), ("BoolExact", BoolExact), - ("TrueClassExact", TrueClassExact), - ("Symbol", Symbol), - ("SymbolSubclass", SymbolSubclass), + ("TrueClass", TrueClass), + ("HeapBasicObject", HeapBasicObject), + ("HeapObject", HeapObject), ("String", String), + ("Subclass", Subclass), ("StringSubclass", StringSubclass), ("StringExact", StringExact), - ("SymbolExact", SymbolExact), + ("Symbol", Symbol), ("StaticSymbol", StaticSymbol), + ("Set", Set), + ("SetSubclass", SetSubclass), + ("SetExact", SetExact), + ("Regexp", Regexp), + ("RegexpSubclass", RegexpSubclass), + ("RegexpExact", RegexpExact), + ("Range", Range), + ("RangeSubclass", RangeSubclass), + ("RangeExact", RangeExact), ("ObjectSubclass", ObjectSubclass), ("ObjectExact", ObjectExact), + ("Numeric", Numeric), + ("NumericSubclass", NumericSubclass), + ("NumericExact", NumericExact), ("NilClass", NilClass), - ("NilClassSubclass", NilClassSubclass), - ("NilClassExact", NilClassExact), - ("Integer", Integer), - ("IntegerSubclass", IntegerSubclass), + ("Module", Module), + ("ModuleSubclass", ModuleSubclass), + ("ModuleExact", ModuleExact), ("Float", Float), - ("FloatExact", FloatExact), ("HeapFloat", HeapFloat), ("Hash", Hash), ("HashSubclass", HashSubclass), ("HashExact", HashExact), ("Flonum", Flonum), - ("FloatSubclass", FloatSubclass), - ("IntegerExact", IntegerExact), + ("Integer", Integer), ("Fixnum", Fixnum), ("FalseClass", FalseClass), - ("FalseClassSubclass", FalseClassSubclass), - ("FalseClassExact", FalseClassExact), ("DynamicSymbol", DynamicSymbol), + ("Class", Class), + ("CallableMethodEntry", CallableMethodEntry), ("CValue", CValue), ("CInt", CInt), ("CUnsigned", CUnsigned), @@ -111,6 +126,7 @@ mod bits { ("CUInt64", CUInt64), ("CUInt32", CUInt32), ("CUInt16", CUInt16), + ("CShape", CShape), ("CPtr", CPtr), ("CNull", CNull), ("CSigned", CSigned), @@ -128,7 +144,7 @@ mod bits { ("ArrayExact", ArrayExact), ("Empty", Empty), ]; - pub const NumTypeBits: u64 = 38; + pub const NumTypeBits: u64 = 45; } pub mod types { use super::*; @@ -151,6 +167,7 @@ pub mod types { pub const CInt8: Type = Type::from_bits(bits::CInt8); pub const CNull: Type = Type::from_bits(bits::CNull); pub const CPtr: Type = Type::from_bits(bits::CPtr); + pub const CShape: Type = Type::from_bits(bits::CShape); pub const CSigned: Type = Type::from_bits(bits::CSigned); pub const CUInt16: Type = Type::from_bits(bits::CUInt16); pub const CUInt32: Type = Type::from_bits(bits::CUInt32); @@ -158,41 +175,79 @@ pub mod types { pub const CUInt8: Type = Type::from_bits(bits::CUInt8); pub const CUnsigned: Type = Type::from_bits(bits::CUnsigned); pub const CValue: Type = Type::from_bits(bits::CValue); + pub const CallableMethodEntry: Type = Type::from_bits(bits::CallableMethodEntry); + pub const Class: Type = Type::from_bits(bits::Class); pub const DynamicSymbol: Type = Type::from_bits(bits::DynamicSymbol); pub const Empty: Type = Type::from_bits(bits::Empty); pub const FalseClass: Type = Type::from_bits(bits::FalseClass); - pub const FalseClassExact: Type = Type::from_bits(bits::FalseClassExact); - pub const FalseClassSubclass: Type = Type::from_bits(bits::FalseClassSubclass); pub const Fixnum: Type = Type::from_bits(bits::Fixnum); pub const Float: Type = Type::from_bits(bits::Float); - pub const FloatExact: Type = Type::from_bits(bits::FloatExact); - pub const FloatSubclass: Type = Type::from_bits(bits::FloatSubclass); pub const Flonum: Type = Type::from_bits(bits::Flonum); pub const Hash: Type = Type::from_bits(bits::Hash); pub const HashExact: Type = Type::from_bits(bits::HashExact); pub const HashSubclass: Type = Type::from_bits(bits::HashSubclass); + pub const HeapBasicObject: Type = Type::from_bits(bits::HeapBasicObject); pub const HeapFloat: Type = Type::from_bits(bits::HeapFloat); + pub const HeapObject: Type = Type::from_bits(bits::HeapObject); pub const Immediate: Type = Type::from_bits(bits::Immediate); pub const Integer: Type = Type::from_bits(bits::Integer); - pub const IntegerExact: Type = Type::from_bits(bits::IntegerExact); - pub const IntegerSubclass: Type = Type::from_bits(bits::IntegerSubclass); + pub const Module: Type = Type::from_bits(bits::Module); + pub const ModuleExact: Type = Type::from_bits(bits::ModuleExact); + pub const ModuleSubclass: Type = Type::from_bits(bits::ModuleSubclass); pub const NilClass: Type = Type::from_bits(bits::NilClass); - pub const NilClassExact: Type = Type::from_bits(bits::NilClassExact); - pub const NilClassSubclass: Type = Type::from_bits(bits::NilClassSubclass); + pub const Numeric: Type = Type::from_bits(bits::Numeric); + pub const NumericExact: Type = Type::from_bits(bits::NumericExact); + pub const NumericSubclass: Type = Type::from_bits(bits::NumericSubclass); pub const Object: Type = Type::from_bits(bits::Object); pub const ObjectExact: Type = Type::from_bits(bits::ObjectExact); pub const ObjectSubclass: Type = Type::from_bits(bits::ObjectSubclass); + pub const Range: Type = Type::from_bits(bits::Range); + pub const RangeExact: Type = Type::from_bits(bits::RangeExact); + pub const RangeSubclass: Type = Type::from_bits(bits::RangeSubclass); + pub const Regexp: Type = Type::from_bits(bits::Regexp); + pub const RegexpExact: Type = Type::from_bits(bits::RegexpExact); + pub const RegexpSubclass: Type = Type::from_bits(bits::RegexpSubclass); pub const RubyValue: Type = Type::from_bits(bits::RubyValue); + pub const Set: Type = Type::from_bits(bits::Set); + pub const SetExact: Type = Type::from_bits(bits::SetExact); + pub const SetSubclass: Type = Type::from_bits(bits::SetSubclass); pub const StaticSymbol: Type = Type::from_bits(bits::StaticSymbol); pub const String: Type = Type::from_bits(bits::String); pub const StringExact: Type = Type::from_bits(bits::StringExact); pub const StringSubclass: Type = Type::from_bits(bits::StringSubclass); pub const Subclass: Type = Type::from_bits(bits::Subclass); pub const Symbol: Type = Type::from_bits(bits::Symbol); - pub const SymbolExact: Type = Type::from_bits(bits::SymbolExact); - pub const SymbolSubclass: Type = Type::from_bits(bits::SymbolSubclass); pub const TrueClass: Type = Type::from_bits(bits::TrueClass); - pub const TrueClassExact: Type = Type::from_bits(bits::TrueClassExact); - pub const TrueClassSubclass: Type = Type::from_bits(bits::TrueClassSubclass); pub const Undef: Type = Type::from_bits(bits::Undef); + pub const ExactBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::ObjectExact, &raw const crate::cruby::rb_cObject), + (bits::BasicObjectExact, &raw const crate::cruby::rb_cBasicObject), + (bits::StringExact, &raw const crate::cruby::rb_cString), + (bits::ArrayExact, &raw const crate::cruby::rb_cArray), + (bits::HashExact, &raw const crate::cruby::rb_cHash), + (bits::RangeExact, &raw const crate::cruby::rb_cRange), + (bits::SetExact, &raw const crate::cruby::rb_cSet), + (bits::RegexpExact, &raw const crate::cruby::rb_cRegexp), + (bits::ModuleExact, &raw const crate::cruby::rb_cModule), + (bits::Class, &raw const crate::cruby::rb_cClass), + (bits::NumericExact, &raw const crate::cruby::rb_cNumeric), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + ]; + pub const InexactBitsAndClass: [(u64, *const VALUE); 10] = [ + (bits::ArraySubclass, &raw const crate::cruby::rb_cArray), + (bits::HashSubclass, &raw const crate::cruby::rb_cHash), + (bits::ModuleSubclass, &raw const crate::cruby::rb_cModule), + (bits::NumericSubclass, &raw const crate::cruby::rb_cNumeric), + (bits::RangeSubclass, &raw const crate::cruby::rb_cRange), + (bits::RegexpSubclass, &raw const crate::cruby::rb_cRegexp), + (bits::SetSubclass, &raw const crate::cruby::rb_cSet), + (bits::StringSubclass, &raw const crate::cruby::rb_cString), + (bits::Object, &raw const crate::cruby::rb_cObject), + (bits::BasicObject, &raw const crate::cruby::rb_cBasicObject), + ]; } diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs index f19c724417..cc6a208bcd 100644 --- a/zjit/src/hir_type/mod.rs +++ b/zjit/src/hir_type/mod.rs @@ -1,9 +1,15 @@ +//! High-level intermediate representation types. + #![allow(non_upper_case_globals)] -use crate::cruby::{Qfalse, Qnil, Qtrue, VALUE, RUBY_T_ARRAY, RUBY_T_STRING, RUBY_T_HASH}; -use crate::cruby::{rb_cInteger, rb_cFloat, rb_cArray, rb_cHash, rb_cString, rb_cSymbol, rb_cObject, rb_cTrueClass, rb_cFalseClass, rb_cNilClass}; +use crate::cruby::{rb_block_param_proxy, Qfalse, Qnil, Qtrue, RUBY_T_ARRAY, RUBY_T_CLASS, RUBY_T_HASH, RUBY_T_MODULE, RUBY_T_STRING, VALUE}; +use crate::cruby::{rb_cInteger, rb_cFloat, rb_cArray, rb_cHash, rb_cString, rb_cSymbol, rb_cRange, rb_cModule, rb_zjit_singleton_class_p}; use crate::cruby::ClassRelationship; use crate::cruby::get_class_name; -use crate::hir::PtrPrintMap; +use crate::cruby::ruby_sym_to_rust_string; +use crate::cruby::rb_mRubyVMFrozenCore; +use crate::cruby::rb_obj_class; +use crate::hir::{Const, PtrPrintMap}; +use crate::profile::ProfiledType; #[derive(Copy, Clone, Debug, PartialEq)] /// Specialization of the type. If we know additional information about the object, we put it here. @@ -68,18 +74,32 @@ fn write_spec(f: &mut std::fmt::Formatter, printer: &TypePrinter) -> std::fmt::R let ty = printer.inner; match ty.spec { Specialization::Any | Specialization::Empty => { Ok(()) }, + Specialization::Object(val) if val == unsafe { rb_mRubyVMFrozenCore } => write!(f, "[VMFrozenCore]"), + Specialization::Object(val) if val == unsafe { rb_block_param_proxy } => write!(f, "[BlockParamProxy]"), + Specialization::Object(val) if ty.is_subtype(types::Symbol) => write!(f, "[:{}]", ruby_sym_to_rust_string(val)), + Specialization::Object(val) if ty.is_subtype(types::Class) => + write!(f, "[{}@{:p}]", get_class_name(val), printer.ptr_map.map_ptr(val.0 as *const std::ffi::c_void)), Specialization::Object(val) => write!(f, "[{}]", val.print(printer.ptr_map)), + // TODO(max): Ensure singleton classes never have Type specialization + Specialization::Type(val) if unsafe { rb_zjit_singleton_class_p(val) } => + write!(f, "[class*:{}@{}]", get_class_name(val), val.print(printer.ptr_map)), Specialization::Type(val) => write!(f, "[class:{}]", get_class_name(val)), - Specialization::TypeExact(val) => write!(f, "[class_exact:{}]", get_class_name(val)), + Specialization::TypeExact(val) if unsafe { rb_zjit_singleton_class_p(val) } => + write!(f, "[class_exact*:{}@{}]", get_class_name(val), val.print(printer.ptr_map)), + Specialization::TypeExact(val) => + write!(f, "[class_exact:{}]", get_class_name(val)), Specialization::Int(val) if ty.is_subtype(types::CBool) => write!(f, "[{}]", val != 0), - Specialization::Int(val) if ty.is_subtype(types::CInt8) => write!(f, "[{}]", (val as i64) >> 56), - Specialization::Int(val) if ty.is_subtype(types::CInt16) => write!(f, "[{}]", (val as i64) >> 48), - Specialization::Int(val) if ty.is_subtype(types::CInt32) => write!(f, "[{}]", (val as i64) >> 32), + Specialization::Int(val) if ty.is_subtype(types::CInt8) => write!(f, "[{}]", (val & u8::MAX as u64) as i8), + Specialization::Int(val) if ty.is_subtype(types::CInt16) => write!(f, "[{}]", (val & u16::MAX as u64) as i16), + Specialization::Int(val) if ty.is_subtype(types::CInt32) => write!(f, "[{}]", (val & u32::MAX as u64) as i32), + Specialization::Int(val) if ty.is_subtype(types::CShape) => + write!(f, "[{:p}]", printer.ptr_map.map_shape(crate::cruby::ShapeId((val & u32::MAX as u64) as u32))), Specialization::Int(val) if ty.is_subtype(types::CInt64) => write!(f, "[{}]", val as i64), - Specialization::Int(val) if ty.is_subtype(types::CUInt8) => write!(f, "[{}]", val >> 56), - Specialization::Int(val) if ty.is_subtype(types::CUInt16) => write!(f, "[{}]", val >> 48), - Specialization::Int(val) if ty.is_subtype(types::CUInt32) => write!(f, "[{}]", val >> 32), - Specialization::Int(val) if ty.is_subtype(types::CUInt64) => write!(f, "[{}]", val), + Specialization::Int(val) if ty.is_subtype(types::CUInt8) => write!(f, "[{}]", val & u8::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt16) => write!(f, "[{}]", val & u16::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt32) => write!(f, "[{}]", val & u32::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt64) => write!(f, "[{val}]"), + Specialization::Int(val) if ty.is_subtype(types::CPtr) => write!(f, "[{}]", Const::CPtr(val as *const u8).print(printer.ptr_map)), Specialization::Int(val) => write!(f, "[{val}]"), Specialization::Double(val) => write!(f, "[{val}]"), } @@ -137,6 +157,22 @@ fn is_hash_exact(val: VALUE) -> bool { val.class_of() == unsafe { rb_cHash } || (val.class_of() == VALUE(0) && val.builtin_type() == RUBY_T_HASH) } +fn is_range_exact(val: VALUE) -> bool { + val.class_of() == unsafe { rb_cRange } +} + +fn is_module_exact(val: VALUE) -> bool { + if val.builtin_type() != RUBY_T_MODULE { + return false; + } + + // For Class and Module instances, `class_of` will return the singleton class of the object. + // Using `rb_obj_class` will give us the actual class of the module so we can check if the + // object is an instance of Module, or an instance of Module subclass. + let klass = unsafe { rb_obj_class(val) }; + klass == unsafe { rb_cModule } +} + impl Type { /// Create a `Type` from the given integer. pub const fn fixnum(val: i64) -> Type { @@ -146,10 +182,50 @@ impl Type { } } + fn bits_from_exact_class(class: VALUE) -> Option<u64> { + types::ExactBitsAndClass + .iter() + .find(|&(_, class_object)| unsafe { **class_object } == class) + .map(|&(bits, _)| bits) + } + + fn bits_from_subclass(class: VALUE) -> Option<u64> { + types::InexactBitsAndClass + .iter() + .find(|&(_, class_object)| class.is_subclass_of(unsafe { **class_object }) == ClassRelationship::Subclass) + // Can't be an immediate if it's a subclass. + .map(|&(bits, _)| bits & !bits::Immediate) + } + + fn from_heap_object(val: VALUE) -> Type { + assert!(!val.special_const_p(), "val should be a heap object"); + let bits = + // GC-hidden types + if is_array_exact(val) { bits::ArrayExact } + else if is_hash_exact(val) { bits::HashExact } + else if is_string_exact(val) { bits::StringExact } + // Singleton classes + else if is_module_exact(val) { bits::ModuleExact } + else if val.builtin_type() == RUBY_T_CLASS { bits::Class } + // Classes that have an immediate/heap split + else if val.class_of() == unsafe { rb_cInteger } { bits::Bignum } + else if val.class_of() == unsafe { rb_cFloat } { bits::HeapFloat } + else if val.class_of() == unsafe { rb_cSymbol } { bits::DynamicSymbol } + else if let Some(bits) = Self::bits_from_exact_class(val.class_of()) { bits } + else if let Some(bits) = Self::bits_from_subclass(val.class_of()) { bits } + else { + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(val.class_of())) + }; + let spec = Specialization::Object(val); + Type { bits, spec } + } + /// Create a `Type` from a Ruby `VALUE`. The type is not guaranteed to have object /// specialization in its `specialization` field (for example, `Qnil` will just be - /// `types::NilClassExact`), but will be available via `ruby_object()`. + /// `types::NilClass`), but will be available via `ruby_object()`. pub fn from_value(val: VALUE) -> Type { + // Immediates if val.fixnum_p() { Type { bits: bits::Fixnum, spec: Specialization::Object(val) } } @@ -160,34 +236,56 @@ impl Type { Type { bits: bits::StaticSymbol, spec: Specialization::Object(val) } } // Singleton objects; don't specialize - else if val == Qnil { types::NilClassExact } - else if val == Qtrue { types::TrueClassExact } - else if val == Qfalse { types::FalseClassExact } - else if val.class_of() == unsafe { rb_cInteger } { - Type { bits: bits::Bignum, spec: Specialization::Object(val) } + else if val == Qnil { types::NilClass } + else if val == Qtrue { types::TrueClass } + else if val == Qfalse { types::FalseClass } + else if val.cme_p() { + // NB: Checking for CME has to happen before looking at class_of because that's not + // valid on imemo. + Type { bits: bits::CallableMethodEntry, spec: Specialization::Object(val) } } - else if val.class_of() == unsafe { rb_cFloat } { - Type { bits: bits::HeapFloat, spec: Specialization::Object(val) } - } - else if val.class_of() == unsafe { rb_cSymbol } { - Type { bits: bits::DynamicSymbol, spec: Specialization::Object(val) } - } - else if is_array_exact(val) { - Type { bits: bits::ArrayExact, spec: Specialization::Object(val) } - } - else if is_hash_exact(val) { - Type { bits: bits::HashExact, spec: Specialization::Object(val) } + else { + Self::from_heap_object(val) } - else if is_string_exact(val) { - Type { bits: bits::StringExact, spec: Specialization::Object(val) } + } + + pub fn from_const(val: Const) -> Type { + match val { + Const::Value(v) => Self::from_value(v), + Const::CBool(v) => Self::from_cbool(v), + Const::CInt8(v) => Self::from_cint(types::CInt8, v as i64), + Const::CInt16(v) => Self::from_cint(types::CInt16, v as i64), + Const::CInt32(v) => Self::from_cint(types::CInt32, v as i64), + Const::CInt64(v) => Self::from_cint(types::CInt64, v), + Const::CUInt8(v) => Self::from_cint(types::CUInt8, v as i64), + Const::CUInt16(v) => Self::from_cint(types::CUInt16, v as i64), + Const::CUInt32(v) => Self::from_cint(types::CUInt32, v as i64), + Const::CShape(v) => Self::from_cint(types::CShape, v.0 as i64), + Const::CUInt64(v) => Self::from_cint(types::CUInt64, v as i64), + Const::CPtr(v) => Self::from_cptr(v), + Const::CDouble(v) => Self::from_double(v), } - else if val.class_of() == unsafe { rb_cObject } { - Type { bits: bits::ObjectExact, spec: Specialization::Object(val) } + } + + pub fn from_profiled_type(val: ProfiledType) -> Type { + if val.is_fixnum() { types::Fixnum } + else if val.is_flonum() { types::Flonum } + else if val.is_static_symbol() { types::StaticSymbol } + else if val.is_nil() { types::NilClass } + else if val.is_true() { types::TrueClass } + else if val.is_false() { types::FalseClass } + else { Self::from_class(val.class()) } + } + + pub fn from_class(class: VALUE) -> Type { + if let Some(bits) = Self::bits_from_exact_class(class) { + return Type::from_bits(bits); } - else { - // TODO(max): Add more cases for inferring type bits from built-in types - Type { bits: bits::BasicObject, spec: Specialization::Object(val) } + if let Some(bits) = Self::bits_from_subclass(class) { + return Type { bits, spec: Specialization::TypeExact(class) } } + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(class)) } /// Private. Only for creating type globals. @@ -213,6 +311,10 @@ impl Type { Type { bits: ty.bits, spec: Specialization::Int(val as u64) } } + pub fn from_cptr(val: *const u8) -> Type { + Type { bits: bits::CPtr, spec: Specialization::Int(val as u64) } + } + /// Create a `Type` (a `CDouble` with double specialization) from a f64. pub fn from_double(val: f64) -> Type { Type { bits: bits::CDouble, spec: Specialization::Double(val) } @@ -225,18 +327,31 @@ impl Type { /// Return true if the value with this type is definitely truthy. pub fn is_known_truthy(&self) -> bool { - !self.could_be(types::NilClassExact) && !self.could_be(types::FalseClassExact) + !self.could_be(types::NilClass) && !self.could_be(types::FalseClass) } /// Return true if the value with this type is definitely falsy. pub fn is_known_falsy(&self) -> bool { - self.is_subtype(types::NilClassExact) || self.is_subtype(types::FalseClassExact) + self.is_subtype(types::NilClass) || self.is_subtype(types::FalseClass) } - /// Top self is the Ruby global object, where top-level method definitions go. Return true if - /// this Type has a Ruby object specialization that is the top-level self. - pub fn is_top_self(&self) -> bool { - self.ruby_object() == Some(unsafe { crate::cruby::rb_vm_top_self() }) + pub fn has_value(&self, val: Const) -> bool { + match (self.spec, val) { + (Specialization::Object(v1), Const::Value(v2)) => v1 == v2, + (Specialization::Int(v1), Const::CBool(v2)) if self.is_subtype(types::CBool) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt8(v2)) if self.is_subtype(types::CInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt16(v2)) if self.is_subtype(types::CInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt32(v2)) if self.is_subtype(types::CInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt64(v2)) if self.is_subtype(types::CInt64) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt8(v2)) if self.is_subtype(types::CUInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt16(v2)) if self.is_subtype(types::CUInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt32(v2)) if self.is_subtype(types::CUInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CShape(v2)) if self.is_subtype(types::CShape) => v1 == (v2.0 as u64), + (Specialization::Int(v1), Const::CUInt64(v2)) if self.is_subtype(types::CUInt64) => v1 == v2, + (Specialization::Int(v1), Const::CPtr(v2)) if self.is_subtype(types::CPtr) => v1 == (v2 as u64), + (Specialization::Double(v1), Const::CDouble(v2)) => v1.to_bits() == v2.to_bits(), + _ => false, + } } /// Return the object specialization, if any. @@ -247,6 +362,27 @@ impl Type { } } + /// Return a Ruby object that needs to be marked on GC. + /// This covers Type and TypeExact unlike ruby_object(). + pub fn gc_object(&self) -> Option<VALUE> { + match self.spec { + Specialization::Type(val) | + Specialization::TypeExact(val) | + Specialization::Object(val) => Some(val), + _ => None, + } + } + + /// Mutable version of gc_object(). + pub fn gc_object_mut(&mut self) -> Option<&mut VALUE> { + match &mut self.spec { + Specialization::Type(val) | + Specialization::TypeExact(val) | + Specialization::Object(val) => Some(val), + _ => None, + } + } + pub fn unspecialized(&self) -> Self { Type { spec: Specialization::Any, ..*self } } @@ -259,23 +395,22 @@ impl Type { } } + pub fn cint64_value(&self) -> Option<i64> { + match (self.is_subtype(types::CInt64), &self.spec) { + (true, Specialization::Int(val)) => Some(*val as i64), + _ => None, + } + } + /// Return true if the Type has object specialization and false otherwise. pub fn ruby_object_known(&self) -> bool { matches!(self.spec, Specialization::Object(_)) } fn is_builtin(class: VALUE) -> bool { - if class == unsafe { rb_cArray } { return true; } - if class == unsafe { rb_cFalseClass } { return true; } - if class == unsafe { rb_cFloat } { return true; } - if class == unsafe { rb_cHash } { return true; } - if class == unsafe { rb_cInteger } { return true; } - if class == unsafe { rb_cNilClass } { return true; } - if class == unsafe { rb_cObject } { return true; } - if class == unsafe { rb_cString } { return true; } - if class == unsafe { rb_cSymbol } { return true; } - if class == unsafe { rb_cTrueClass } { return true; } - false + types::ExactBitsAndClass + .iter() + .any(|&(_, class_object)| unsafe { *class_object } == class) } /// Union both types together, preserving specialization if possible. @@ -365,23 +500,16 @@ impl Type { } /// Return a pointer to the Ruby class that an object of this Type would have at run-time, if - /// known. This includes classes for HIR types such as ArrayExact or NilClassExact, which have + /// known. This includes classes for HIR types such as ArrayExact or NilClass, which have /// canonical Type representations that lack an explicit specialization in their `spec` fields. pub fn runtime_exact_ruby_class(&self) -> Option<VALUE> { if let Some(val) = self.exact_ruby_class() { return Some(val); } - if self.is_subtype(types::ArrayExact) { return Some(unsafe { rb_cArray }); } - if self.is_subtype(types::FalseClassExact) { return Some(unsafe { rb_cFalseClass }); } - if self.is_subtype(types::FloatExact) { return Some(unsafe { rb_cFloat }); } - if self.is_subtype(types::HashExact) { return Some(unsafe { rb_cHash }); } - if self.is_subtype(types::IntegerExact) { return Some(unsafe { rb_cInteger }); } - if self.is_subtype(types::NilClassExact) { return Some(unsafe { rb_cNilClass }); } - if self.is_subtype(types::ObjectExact) { return Some(unsafe { rb_cObject }); } - if self.is_subtype(types::StringExact) { return Some(unsafe { rb_cString }); } - if self.is_subtype(types::SymbolExact) { return Some(unsafe { rb_cSymbol }); } - if self.is_subtype(types::TrueClassExact) { return Some(unsafe { rb_cTrueClass }); } - None + types::ExactBitsAndClass + .iter() + .find(|&(bits, _)| self.is_subtype(Type::from_bits(*bits))) + .map(|&(_, class_object)| unsafe { *class_object }) } /// Check bit equality of two `Type`s. Do not use! You are probably looking for [`Type::is_subtype`]. @@ -411,13 +539,29 @@ impl Type { } } - fn is_immediate(&self) -> bool { + pub fn is_immediate(&self) -> bool { self.is_subtype(types::Immediate) } - pub fn print(self, ptr_map: &PtrPrintMap) -> TypePrinter { + pub fn print(self, ptr_map: &PtrPrintMap) -> TypePrinter<'_> { TypePrinter { inner: self, ptr_map } } + + pub fn num_bits(&self) -> u8 { + self.num_bytes() * crate::cruby::BITS_PER_BYTE as u8 + } + + pub fn num_bytes(&self) -> u8 { + if self.is_subtype(types::CUInt8) || self.is_subtype(types::CInt8) { return 1; } + if self.is_subtype(types::CUInt16) || self.is_subtype(types::CInt16) { return 2; } + if self.is_subtype(types::CUInt32) || self.is_subtype(types::CInt32) { return 4; } + if self.is_subtype(types::CShape) { + use crate::cruby::{SHAPE_ID_NUM_BITS, BITS_PER_BYTE}; + return (SHAPE_ID_NUM_BITS as usize / BITS_PER_BYTE).try_into().unwrap(); + } + // CUInt64, CInt64, CPtr, CNull, CDouble, or anything else defaults to 8 bytes + crate::cruby::SIZEOF_VALUE as u8 + } } #[cfg(test)] @@ -429,6 +573,11 @@ mod tests { use crate::cruby::rb_hash_new; use crate::cruby::rb_float_new; use crate::cruby::define_class; + use crate::cruby::rb_cObject; + use crate::cruby::rb_cSet; + use crate::cruby::rb_cTrueClass; + use crate::cruby::rb_cFalseClass; + use crate::cruby::rb_cNilClass; #[track_caller] fn assert_bit_equal(left: Type, right: Type) { @@ -449,7 +598,7 @@ mod tests { #[test] fn empty_is_subtype_of_everything() { // Spot check a few cases - assert_subtype(types::Empty, types::NilClassExact); + assert_subtype(types::Empty, types::NilClass); assert_subtype(types::Empty, types::Array); assert_subtype(types::Empty, types::Object); assert_subtype(types::Empty, types::CUInt16); @@ -461,7 +610,7 @@ mod tests { #[test] fn everything_is_a_subtype_of_any() { // Spot check a few cases - assert_subtype(types::NilClassExact, types::Any); + assert_subtype(types::NilClass, types::Any); assert_subtype(types::Array, types::Any); assert_subtype(types::Object, types::Any); assert_subtype(types::CUInt16, types::Any); @@ -471,31 +620,63 @@ mod tests { } #[test] + fn from_const() { + let cint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int(12)); + assert_eq!(format!("{}", cint32), "CInt32[12]"); + + let cint32 = Type::from_const(Const::CInt32(-12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int((-12i64) as u64)); + assert_eq!(format!("{}", cint32), "CInt32[-12]"); + + let cuint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cuint32, types::CInt32); + assert_eq!(cuint32.spec, Specialization::Int(12)); + + let cuint32 = Type::from_const(Const::CUInt32(0xffffffff)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xffffffff)); + assert_eq!(format!("{}", cuint32), "CUInt32[4294967295]"); + + let cuint32 = Type::from_const(Const::CUInt32(0xc00087)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xc00087)); + assert_eq!(format!("{}", cuint32), "CUInt32[12583047]"); + } + + #[test] fn integer() { assert_subtype(Type::fixnum(123), types::Fixnum); assert_subtype(Type::fixnum(123), Type::fixnum(123)); assert_not_subtype(Type::fixnum(123), Type::fixnum(200)); assert_subtype(Type::from_value(VALUE::fixnum_from_usize(123)), types::Fixnum); - assert_subtype(types::Fixnum, types::IntegerExact); - assert_subtype(types::Bignum, types::IntegerExact); - assert_subtype(types::IntegerExact, types::Integer); - assert_subtype(types::IntegerSubclass, types::Integer); + assert_subtype(types::Fixnum, types::Integer); + assert_subtype(types::Bignum, types::Integer); } #[test] fn float() { - assert_subtype(types::Flonum, types::FloatExact); - assert_subtype(types::HeapFloat, types::FloatExact); - assert_subtype(types::FloatExact, types::Float); - assert_subtype(types::FloatSubclass, types::Float); + assert_subtype(types::Flonum, types::Float); + assert_subtype(types::HeapFloat, types::Float); + } + + #[test] + fn numeric() { + assert_subtype(types::Integer, types::Numeric); + assert_subtype(types::Float, types::Numeric); + assert_subtype(types::Float.union(types::Integer), types::Numeric); + assert_bit_equal(types::Float + .union(types::Integer) + .union(types::NumericExact) + .union(types::NumericSubclass), types::Numeric); } #[test] fn symbol() { - assert_subtype(types::StaticSymbol, types::SymbolExact); - assert_subtype(types::DynamicSymbol, types::SymbolExact); - assert_subtype(types::SymbolExact, types::Symbol); - assert_subtype(types::SymbolSubclass, types::Symbol); + assert_subtype(types::StaticSymbol, types::Symbol); + assert_subtype(types::DynamicSymbol, types::Symbol); } #[test] @@ -503,12 +684,10 @@ mod tests { assert_subtype(Type::fixnum(123), types::Immediate); assert_subtype(types::Fixnum, types::Immediate); assert_not_subtype(types::Bignum, types::Immediate); - assert_subtype(types::NilClassExact, types::Immediate); - assert_subtype(types::TrueClassExact, types::Immediate); - assert_subtype(types::FalseClassExact, types::Immediate); - assert_not_subtype(types::NilClassSubclass, types::Immediate); - assert_not_subtype(types::TrueClassSubclass, types::Immediate); - assert_not_subtype(types::FalseClassSubclass, types::Immediate); + assert_not_subtype(types::Integer, types::Immediate); + assert_subtype(types::NilClass, types::Immediate); + assert_subtype(types::TrueClass, types::Immediate); + assert_subtype(types::FalseClass, types::Immediate); assert_subtype(types::StaticSymbol, types::Immediate); assert_not_subtype(types::DynamicSymbol, types::Immediate); assert_subtype(types::Flonum, types::Immediate); @@ -516,57 +695,138 @@ mod tests { } #[test] + fn heap_basic_object() { + assert_not_subtype(Type::fixnum(123), types::HeapBasicObject); + assert_not_subtype(types::Fixnum, types::HeapBasicObject); + assert_subtype(types::Bignum, types::HeapBasicObject); + assert_not_subtype(types::Integer, types::HeapBasicObject); + assert_not_subtype(types::NilClass, types::HeapBasicObject); + assert_not_subtype(types::TrueClass, types::HeapBasicObject); + assert_not_subtype(types::FalseClass, types::HeapBasicObject); + assert_not_subtype(types::StaticSymbol, types::HeapBasicObject); + assert_subtype(types::DynamicSymbol, types::HeapBasicObject); + assert_not_subtype(types::Flonum, types::HeapBasicObject); + assert_subtype(types::HeapFloat, types::HeapBasicObject); + assert_not_subtype(types::BasicObject, types::HeapBasicObject); + assert_not_subtype(types::Object, types::HeapBasicObject); + assert_not_subtype(types::Immediate, types::HeapBasicObject); + assert_not_subtype(types::HeapBasicObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapBasicObject); + assert_subtype(right, types::HeapBasicObject); + assert_subtype(left.union(right), types::HeapBasicObject); + }); + } + + #[test] + fn heap_object() { + assert_not_subtype(Type::fixnum(123), types::HeapObject); + assert_not_subtype(types::Fixnum, types::HeapObject); + assert_subtype(types::Bignum, types::HeapObject); + assert_not_subtype(types::Integer, types::HeapObject); + assert_not_subtype(types::NilClass, types::HeapObject); + assert_not_subtype(types::TrueClass, types::HeapObject); + assert_not_subtype(types::FalseClass, types::HeapObject); + assert_not_subtype(types::StaticSymbol, types::HeapObject); + assert_subtype(types::DynamicSymbol, types::HeapObject); + assert_not_subtype(types::Flonum, types::HeapObject); + assert_subtype(types::HeapFloat, types::HeapObject); + assert_not_subtype(types::BasicObject, types::HeapObject); + assert_not_subtype(types::Object, types::HeapObject); + assert_not_subtype(types::Immediate, types::HeapObject); + assert_not_subtype(types::HeapObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapObject); + assert_subtype(right, types::HeapObject); + assert_subtype(left.union(right), types::HeapObject); + }); + } + + #[test] fn fixnum_has_ruby_object() { assert_eq!(Type::fixnum(3).ruby_object(), Some(VALUE::fixnum_from_usize(3))); assert_eq!(types::Fixnum.ruby_object(), None); - assert_eq!(types::IntegerExact.ruby_object(), None); assert_eq!(types::Integer.ruby_object(), None); } #[test] fn singletons_do_not_have_ruby_object() { assert_eq!(Type::from_value(Qnil).ruby_object(), None); - assert_eq!(types::NilClassExact.ruby_object(), None); + assert_eq!(types::NilClass.ruby_object(), None); assert_eq!(Type::from_value(Qtrue).ruby_object(), None); - assert_eq!(types::TrueClassExact.ruby_object(), None); + assert_eq!(types::TrueClass.ruby_object(), None); assert_eq!(Type::from_value(Qfalse).ruby_object(), None); - assert_eq!(types::FalseClassExact.ruby_object(), None); + assert_eq!(types::FalseClass.ruby_object(), None); } #[test] fn integer_has_exact_ruby_class() { - assert_eq!(Type::fixnum(3).exact_ruby_class(), Some(unsafe { rb_cInteger }.into())); + assert_eq!(Type::fixnum(3).exact_ruby_class(), Some(unsafe { rb_cInteger })); assert_eq!(types::Fixnum.exact_ruby_class(), None); - assert_eq!(types::IntegerExact.exact_ruby_class(), None); assert_eq!(types::Integer.exact_ruby_class(), None); } #[test] fn singletons_do_not_have_exact_ruby_class() { assert_eq!(Type::from_value(Qnil).exact_ruby_class(), None); - assert_eq!(types::NilClassExact.exact_ruby_class(), None); + assert_eq!(types::NilClass.exact_ruby_class(), None); assert_eq!(Type::from_value(Qtrue).exact_ruby_class(), None); - assert_eq!(types::TrueClassExact.exact_ruby_class(), None); + assert_eq!(types::TrueClass.exact_ruby_class(), None); assert_eq!(Type::from_value(Qfalse).exact_ruby_class(), None); - assert_eq!(types::FalseClassExact.exact_ruby_class(), None); + assert_eq!(types::FalseClass.exact_ruby_class(), None); } #[test] fn singletons_do_not_have_ruby_class() { assert_eq!(Type::from_value(Qnil).inexact_ruby_class(), None); - assert_eq!(types::NilClassExact.inexact_ruby_class(), None); + assert_eq!(types::NilClass.inexact_ruby_class(), None); assert_eq!(Type::from_value(Qtrue).inexact_ruby_class(), None); - assert_eq!(types::TrueClassExact.inexact_ruby_class(), None); + assert_eq!(types::TrueClass.inexact_ruby_class(), None); assert_eq!(Type::from_value(Qfalse).inexact_ruby_class(), None); - assert_eq!(types::FalseClassExact.inexact_ruby_class(), None); + assert_eq!(types::FalseClass.inexact_ruby_class(), None); + } + + #[test] + fn from_class() { + crate::cruby::with_rubyvm(|| { + assert_bit_equal(Type::from_class(unsafe { rb_cInteger }), types::Integer); + assert_bit_equal(Type::from_class(unsafe { rb_cString }), types::StringExact); + assert_bit_equal(Type::from_class(unsafe { rb_cArray }), types::ArrayExact); + assert_bit_equal(Type::from_class(unsafe { rb_cHash }), types::HashExact); + assert_bit_equal(Type::from_class(unsafe { rb_cNilClass }), types::NilClass); + assert_bit_equal(Type::from_class(unsafe { rb_cTrueClass }), types::TrueClass); + assert_bit_equal(Type::from_class(unsafe { rb_cFalseClass }), types::FalseClass); + let c_class = define_class("C", unsafe { rb_cObject }); + assert_bit_equal(Type::from_class(c_class), Type { bits: bits::HeapObject, spec: Specialization::TypeExact(c_class) }); + }); } #[test] fn integer_has_ruby_class() { - assert_eq!(Type::fixnum(3).inexact_ruby_class(), Some(unsafe { rb_cInteger }.into())); - assert_eq!(types::Fixnum.inexact_ruby_class(), None); - assert_eq!(types::IntegerExact.inexact_ruby_class(), None); - assert_eq!(types::Integer.inexact_ruby_class(), None); + crate::cruby::with_rubyvm(|| { + assert_eq!(Type::fixnum(3).inexact_ruby_class(), Some(unsafe { rb_cInteger })); + assert_eq!(types::Fixnum.inexact_ruby_class(), None); + assert_eq!(types::Integer.inexact_ruby_class(), None); + }); + } + + #[test] + fn set() { + assert_subtype(types::SetExact, types::Set); + assert_subtype(types::SetSubclass, types::Set); + } + + #[test] + fn set_has_ruby_class() { + crate::cruby::with_rubyvm(|| { + assert_eq!(types::SetExact.runtime_exact_ruby_class(), Some(unsafe { rb_cSet })); + assert_eq!(types::Set.runtime_exact_ruby_class(), None); + assert_eq!(types::SetSubclass.runtime_exact_ruby_class(), None); + }); } #[test] @@ -584,7 +844,6 @@ mod tests { assert_eq!(format!("{}", Type::from_cbool(false)), "CBool[false]"); assert_eq!(format!("{}", types::Fixnum), "Fixnum"); assert_eq!(format!("{}", types::Integer), "Integer"); - assert_eq!(format!("{}", types::IntegerExact), "IntegerExact"); } #[test] @@ -602,12 +861,10 @@ mod tests { #[test] fn union_bits_subtype() { - assert_bit_equal(types::Fixnum.union(types::IntegerExact), types::IntegerExact); assert_bit_equal(types::Fixnum.union(types::Integer), types::Integer); assert_bit_equal(types::Fixnum.union(types::Object), types::Object); assert_bit_equal(Type::fixnum(3).union(types::Fixnum), types::Fixnum); - assert_bit_equal(types::IntegerExact.union(types::Fixnum), types::IntegerExact); assert_bit_equal(types::Integer.union(types::Fixnum), types::Integer); assert_bit_equal(types::Object.union(types::Fixnum), types::Object); assert_bit_equal(types::Fixnum.union(Type::fixnum(3)), types::Fixnum); @@ -683,6 +940,30 @@ mod tests { } #[test] + fn cme() { + use crate::cruby::{rb_callable_method_entry, ID}; + crate::cruby::with_rubyvm(|| { + let cme = unsafe { rb_callable_method_entry(rb_cInteger, ID!(to_s)) }; + assert!(!cme.is_null()); + let cme_value: VALUE = cme.into(); + let ty = Type::from_value(cme_value); + assert_subtype(ty, types::CallableMethodEntry); + assert!(ty.ruby_object_known()); + }); + } + + #[test] + fn string_subclass_is_string_subtype() { + crate::cruby::with_rubyvm(|| { + assert_subtype(types::StringExact, types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::StringExact); + let c_class = define_class("C", unsafe { rb_cString }); + assert_subtype(Type::from_class(c_class), types::String); + }); + } + + #[test] fn union_specialized_with_no_relation_returns_unspecialized() { crate::cruby::with_rubyvm(|| { let string = Type::from_value(rust_str_to_ruby("hello")); @@ -703,4 +984,80 @@ mod tests { assert_bit_equal(d_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); }); } + + #[test] + fn has_value() { + // With known values + crate::cruby::with_rubyvm(|| { + let a = rust_str_to_sym("a"); + let b = rust_str_to_sym("b"); + let ty = Type::from_value(a); + assert!(ty.has_value(Const::Value(a))); + assert!(!ty.has_value(Const::Value(b))); + }); + + let true_ty = Type::from_cbool(true); + assert!(true_ty.has_value(Const::CBool(true))); + assert!(!true_ty.has_value(Const::CBool(false))); + + let int8_ty = Type::from_cint(types::CInt8, 42); + assert!(int8_ty.has_value(Const::CInt8(42))); + assert!(!int8_ty.has_value(Const::CInt8(-1))); + let neg_int8_ty = Type::from_cint(types::CInt8, -1); + assert!(neg_int8_ty.has_value(Const::CInt8(-1))); + + let int16_ty = Type::from_cint(types::CInt16, 1000); + assert!(int16_ty.has_value(Const::CInt16(1000))); + assert!(!int16_ty.has_value(Const::CInt16(2000))); + + let int32_ty = Type::from_cint(types::CInt32, 100000); + assert!(int32_ty.has_value(Const::CInt32(100000))); + assert!(!int32_ty.has_value(Const::CInt32(-100000))); + + let int64_ty = Type::from_cint(types::CInt64, i64::MAX); + assert!(int64_ty.has_value(Const::CInt64(i64::MAX))); + assert!(!int64_ty.has_value(Const::CInt64(0))); + + let uint8_ty = Type::from_cint(types::CUInt8, u8::MAX as i64); + assert!(uint8_ty.has_value(Const::CUInt8(u8::MAX))); + assert!(!uint8_ty.has_value(Const::CUInt8(0))); + + let uint16_ty = Type::from_cint(types::CUInt16, u16::MAX as i64); + assert!(uint16_ty.has_value(Const::CUInt16(u16::MAX))); + assert!(!uint16_ty.has_value(Const::CUInt16(1))); + + let uint32_ty = Type::from_cint(types::CUInt32, u32::MAX as i64); + assert!(uint32_ty.has_value(Const::CUInt32(u32::MAX))); + assert!(!uint32_ty.has_value(Const::CUInt32(42))); + + let uint64_ty = Type::from_cint(types::CUInt64, i64::MAX); + assert!(uint64_ty.has_value(Const::CUInt64(i64::MAX as u64))); + assert!(!uint64_ty.has_value(Const::CUInt64(123))); + + let shape_ty = Type::from_cint(types::CShape, 0x1234); + assert!(shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + assert!(!shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x5678)))); + + let ptr = 0x1000 as *const u8; + let ptr_ty = Type::from_cptr(ptr); + assert!(ptr_ty.has_value(Const::CPtr(ptr))); + assert!(!ptr_ty.has_value(Const::CPtr(0x2000 as *const u8))); + + let double_ty = Type::from_double(std::f64::consts::PI); + assert!(double_ty.has_value(Const::CDouble(std::f64::consts::PI))); + assert!(!double_ty.has_value(Const::CDouble(3.123))); + + let nan_ty = Type::from_double(f64::NAN); + assert!(nan_ty.has_value(Const::CDouble(f64::NAN))); + + // Mismatched types + assert!(!int8_ty.has_value(Const::CInt16(42))); + assert!(!int16_ty.has_value(Const::CInt32(1000))); + assert!(!uint8_ty.has_value(Const::CInt8(-1i8))); + + // Wrong specialization (unknown value) + assert!(!types::CInt8.has_value(Const::CInt8(42))); + assert!(!types::CBool.has_value(Const::CBool(true))); + assert!(!types::CShape.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + } } diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index 77fd78d95e..d183eb18ab 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -1,6 +1,71 @@ -use std::collections::HashSet; +//! Code invalidation and patching for speculative optimizations. -use crate::{cruby::{ruby_basic_operators, IseqPtr, RedefinitionFlag}, state::ZJITState, state::zjit_enabled_p}; +use std::{collections::{HashMap, HashSet}, mem}; + +use crate::{backend::lir::{Assembler, asm_comment}, cruby::{ID, IseqPtr, RedefinitionFlag, VALUE, iseq_name, rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock}, hir::Invariant, options::debug, state::{ZJITState, zjit_enabled_p}, virtualmem::CodePtr}; +use crate::payload::{IseqVersionRef, IseqStatus, get_or_create_iseq_payload}; +use crate::codegen::{MAX_ISEQ_VERSIONS, gen_iseq_call}; +use crate::cruby::{rb_iseq_reset_jit_func, iseq_get_location}; +use crate::stats::with_time_stat; +use crate::stats::Counter::invalidation_time_ns; +use crate::gc::remove_gc_offsets; + +macro_rules! compile_patch_points { + ($cb:expr, $patch_points:expr, $($comment_args:tt)*) => { + with_time_stat(invalidation_time_ns, || { + for patch_point in $patch_points { + let written_range = $cb.with_write_ptr(patch_point.patch_point_ptr, |cb| { + let mut asm = Assembler::new(); + asm_comment!(asm, $($comment_args)*); + asm.jmp(patch_point.side_exit_ptr.into()); + asm.compile(cb).expect("can write existing code"); + }); + // Stop marking GC offsets corrupted by the jump instruction + remove_gc_offsets(patch_point.version, &written_range); + + // If the ISEQ doesn't have max versions, invalidate this version. + let mut version = patch_point.version; + let iseq = unsafe { version.as_ref() }.iseq; + if !iseq.is_null() { + let payload = get_or_create_iseq_payload(iseq); + if unsafe { version.as_ref() }.status != IseqStatus::Invalidated && payload.versions.len() < MAX_ISEQ_VERSIONS { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + unsafe { rb_iseq_reset_jit_func(version.as_ref().iseq) }; + + // Recompile JIT-to-JIT calls into the invalidated ISEQ + for incoming in unsafe { version.as_ref() }.incoming.iter() { + if let Err(err) = gen_iseq_call($cb, incoming) { + debug!("{err:?}: gen_iseq_call failed on PatchPoint: {}", iseq_get_location(incoming.iseq.get(), 0)); + } + } + } + } + } + }); + }; +} + +/// When a PatchPoint is invalidated, it generates a jump instruction from `from` to `to`. +#[derive(Debug, Eq, Hash, PartialEq)] +struct PatchPoint { + /// Code pointer to be invalidated + patch_point_ptr: CodePtr, + /// Code pointer to a side exit + side_exit_ptr: CodePtr, + /// ISEQ version to be invalidated + version: IseqVersionRef, +} + +impl PatchPoint { + /// PatchPointer constructor + fn new(patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, version: IseqVersionRef) -> PatchPoint { + Self { + patch_point_ptr, + side_exit_ptr, + version, + } + } +} /// Used to track all of the various block references that contain assumptions /// about the state of the virtual machine. @@ -9,27 +74,132 @@ pub struct Invariants { /// Set of ISEQs that are known to escape EP ep_escape_iseqs: HashSet<IseqPtr>, - /// Set of ISEQs whose JIT code assumes that it doesn't escape EP - no_ep_escape_iseqs: HashSet<IseqPtr>, + /// Map from ISEQ that's assumed to not escape EP to a set of patch points + no_ep_escape_iseq_patch_points: HashMap<IseqPtr, HashSet<PatchPoint>>, + + /// Map from a class and its associated basic operator to a set of patch points + bop_patch_points: HashMap<(RedefinitionFlag, ruby_basic_operators), HashSet<PatchPoint>>, + + /// Map from CME to patch points that assume the method hasn't been redefined + cme_patch_points: HashMap<*const rb_callable_method_entry_t, HashSet<PatchPoint>>, + + /// Map from constant ID to patch points that assume the constant hasn't been redefined + constant_state_patch_points: HashMap<ID, HashSet<PatchPoint>>, + + /// Set of patch points that assume that the TracePoint is not enabled + no_trace_point_patch_points: HashSet<PatchPoint>, + + /// Set of patch points that assume that the interpreter is running with only one ractor + single_ractor_patch_points: HashSet<PatchPoint>, + + /// Map from a class to a set of patch points that assume objects of the class + /// will have no singleton class. + no_singleton_class_patch_points: HashMap<VALUE, HashSet<PatchPoint>>, +} + +impl Invariants { + /// Update object references in Invariants + pub fn update_references(&mut self) { + self.update_ep_escape_iseqs(); + self.update_no_ep_escape_iseq_patch_points(); + self.update_cme_patch_points(); + self.update_no_singleton_class_patch_points(); + } + + /// Forget an ISEQ when freeing it. We need to because a) if the address is reused, we'd be + /// tracking the wrong object b) dead VALUEs in the table can means we risk passing invalid + /// VALUEs to `rb_gc_location()`. + pub fn forget_iseq(&mut self, iseq: IseqPtr) { + // Why not patch the patch points? If the ISEQ is dead then the GC also proved that all + // generated code referencing the ISEQ are unreachable. We mark the ISEQs baked into + // generated code. + self.ep_escape_iseqs.remove(&iseq); + self.no_ep_escape_iseq_patch_points.remove(&iseq); + } + + /// Forget a CME when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_cme(&mut self, cme: *const rb_callable_method_entry_t) { + self.cme_patch_points.remove(&cme); + } + + /// Forget a class when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_klass(&mut self, klass: VALUE) { + self.no_singleton_class_patch_points.remove(&klass); + } + + /// Update ISEQ references in Invariants::ep_escape_iseqs + fn update_ep_escape_iseqs(&mut self) { + let updated = std::mem::take(&mut self.ep_escape_iseqs) + .into_iter() + .map(|iseq| unsafe { rb_gc_location(iseq.into()) }.as_iseq()) + .collect(); + self.ep_escape_iseqs = updated; + } + + /// Update ISEQ references in Invariants::no_ep_escape_iseq_patch_points + fn update_no_ep_escape_iseq_patch_points(&mut self) { + let updated = std::mem::take(&mut self.no_ep_escape_iseq_patch_points) + .into_iter() + .map(|(iseq, patch_points)| { + let new_iseq = unsafe { rb_gc_location(iseq.into()) }; + (new_iseq.as_iseq(), patch_points) + }) + .collect(); + self.no_ep_escape_iseq_patch_points = updated; + } + + fn update_cme_patch_points(&mut self) { + let updated_cme_patch_points = std::mem::take(&mut self.cme_patch_points) + .into_iter() + .map(|(cme, patch_points)| { + let new_cme = unsafe { rb_gc_location(cme.into()) }; + (new_cme.as_cme(), patch_points) + }) + .collect(); + self.cme_patch_points = updated_cme_patch_points; + } + + fn update_no_singleton_class_patch_points(&mut self) { + let updated_no_singleton_class_patch_points = std::mem::take(&mut self.no_singleton_class_patch_points) + .into_iter() + .map(|(klass, patch_points)| { + let new_klass = unsafe { rb_gc_location(klass) }; + (new_klass, patch_points) + }) + .collect(); + self.no_singleton_class_patch_points = updated_no_singleton_class_patch_points; + } } /// Called when a basic operator is redefined. Note that all the blocks assuming /// the stability of different operators are invalidated together and we don't /// do fine-grained tracking. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_bop_redefined(_klass: RedefinitionFlag, _bop: ruby_basic_operators) { +pub extern "C" fn rb_zjit_bop_redefined(klass: RedefinitionFlag, bop: ruby_basic_operators) { // If ZJIT isn't enabled, do nothing if !zjit_enabled_p() { return; } - unimplemented!("Invalidation on BOP redefinition is not implemented yet"); + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + if let Some(patch_points) = invariants.bop_patch_points.get(&(klass, bop)) { + let cb = ZJITState::get_code_block(); + let bop = Invariant::BOPRedefined { klass, bop }; + debug!("BOP is redefined: {}", bop); + + // Invalidate all patch points for this BOP + compile_patch_points!(cb, patch_points, "BOP is redefined: {}", bop); + + cb.mark_all_executable(); + } + }); } /// Invalidate blocks for a given ISEQ that assumes environment pointer is /// equal to base pointer. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_invalidate_ep_is_bp(iseq: IseqPtr) { +pub extern "C" fn rb_zjit_invalidate_no_ep_escape(iseq: IseqPtr) { // Skip tracking EP escapes on boot. We don't need to invalidate anything during boot. if !ZJITState::has_instance() { return; @@ -40,18 +210,266 @@ pub extern "C" fn rb_zjit_invalidate_ep_is_bp(iseq: IseqPtr) { invariants.ep_escape_iseqs.insert(iseq); // If the ISEQ has been compiled assuming it doesn't escape EP, invalidate the JIT code. - if invariants.no_ep_escape_iseqs.contains(&iseq) { - unimplemented!("Invalidation on EP escape is not implemented yet"); + if let Some(patch_points) = invariants.no_ep_escape_iseq_patch_points.get(&iseq) { + debug!("EP is escaped: {}", iseq_name(iseq)); + + // Invalidate the patch points for this ISEQ + let cb = ZJITState::get_code_block(); + compile_patch_points!(cb, patch_points, "EP is escaped: {}", iseq_name(iseq)); + + cb.mark_all_executable(); } } /// Track that JIT code for a ISEQ will assume that base pointer is equal to environment pointer. -pub fn track_no_ep_escape_assumption(iseq: IseqPtr) { +pub fn track_no_ep_escape_assumption( + iseq: IseqPtr, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { let invariants = ZJITState::get_invariants(); - invariants.no_ep_escape_iseqs.insert(iseq); + invariants.no_ep_escape_iseq_patch_points.entry(iseq).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); } /// Returns true if a given ISEQ has previously escaped environment pointer. pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { ZJITState::get_invariants().ep_escape_iseqs.contains(&iseq) } + +/// Track a patch point for a basic operator in a given class. +pub fn track_bop_assumption( + klass: RedefinitionFlag, + bop: ruby_basic_operators, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.bop_patch_points.entry((klass, bop)).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Track a patch point for a callable method entry (CME). +pub fn track_cme_assumption( + cme: *const rb_callable_method_entry_t, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.cme_patch_points.entry(cme).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Track a patch point for each constant name in a constant path assumption. +pub fn track_stable_constant_names_assumption( + idlist: *const ID, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + + let mut idx = 0; + loop { + let id = unsafe { *idlist.wrapping_add(idx) }; + if id.0 == 0 { + break; + } + + invariants.constant_state_patch_points.entry(id).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); + + idx += 1; + } +} + +/// Track a patch point for objects of a given class will have no singleton class. +pub fn track_no_singleton_class_assumption( + klass: VALUE, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_singleton_class_patch_points.entry(klass).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Called when a method is redefined. Invalidates all JIT code that depends on the CME. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_cme_invalidate(cme: *const rb_callable_method_entry_t) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + // Get the CMD's jumps and remove the entry from the map as it has been invalidated + if let Some(patch_points) = invariants.cme_patch_points.remove(&cme) { + let cb = ZJITState::get_code_block(); + debug!("CME is invalidated: {:?}", cme); + + // Invalidate all patch points for this CME + compile_patch_points!(cb, patch_points, "CME is invalidated: {:?}", cme); + + cb.mark_all_executable(); + } + }); +} + +/// Called when a constant is redefined. Invalidates all JIT code that depends on the constant. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_constant_state_changed(id: ID) { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + if let Some(patch_points) = invariants.constant_state_patch_points.get(&id) { + let cb = ZJITState::get_code_block(); + debug!("Constant state changed: {:?}", id); + + // Invalidate all patch points for this constant ID + compile_patch_points!(cb, patch_points, "Constant state changed: {:?}", id); + + cb.mark_all_executable(); + } + }); +} + +/// Track the JIT code that assumes that the interpreter is running with only one ractor +pub fn track_single_ractor_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.single_ractor_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Callback for when Ruby is about to spawn a ractor. In that case we need to +/// invalidate every block that is assuming single ractor mode. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_before_ractor_spawn() { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().single_ractor_patch_points); + + // Invalidate all patch points for single ractor mode + compile_patch_points!(cb, patch_points, "Another ractor spawned, invalidating single ractor mode assumption"); + + cb.mark_all_executable(); + }); +} + +pub fn track_no_trace_point_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_trace_point_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_tracing_invalidate_all() { + use crate::payload::{get_or_create_iseq_payload, IseqStatus}; + use crate::cruby::for_each_iseq; + + if !zjit_enabled_p() { + return; + } + + // Stop other ractors since we are going to patch machine code. + with_vm_lock(src_loc!(), || { + debug!("Invalidating all ZJIT compiled code due to TracePoint"); + + for_each_iseq(|iseq| { + let payload = get_or_create_iseq_payload(iseq); + + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + } + unsafe { rb_iseq_reset_jit_func(iseq) }; + }); + + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().no_trace_point_patch_points); + + compile_patch_points!(cb, patch_points, "TracePoint is enabled, invalidating no TracePoint assumption"); + + cb.mark_all_executable(); + }); +} + +/// Returns true if we've seen a singleton class of a given class since boot. +/// This is used to avoid an invalidation loop where we repeatedly compile code +/// that assumes no singleton class, only to have it invalidated. +pub fn has_singleton_class_of(klass: VALUE) -> bool { + ZJITState::get_invariants() + .no_singleton_class_patch_points + .get(&klass) + .map_or(false, |patch_points| patch_points.is_empty()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_no_singleton_class(klass: VALUE) { + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + match invariants.no_singleton_class_patch_points.get_mut(&klass) { + Some(patch_points) => { + // Invalidate existing patch points and let has_singleton_class_of() + // return true when they are compiled again + let patch_points = mem::take(patch_points); + if !patch_points.is_empty() { + let cb = ZJITState::get_code_block(); + debug!("Singleton class created for {:?}", klass); + compile_patch_points!(cb, patch_points, "Singleton class created for {:?}", klass); + cb.mark_all_executable(); + } + } + None => { + // Let has_singleton_class_of() return true for this class + invariants.no_singleton_class_patch_points.insert(klass, HashSet::new()); + } + } + }); +} diff --git a/zjit/src/json.rs b/zjit/src/json.rs new file mode 100644 index 0000000000..fa4b216821 --- /dev/null +++ b/zjit/src/json.rs @@ -0,0 +1,700 @@ +//! Single file JSON serializer for iongraph output of ZJIT HIR. + +use std::{ + fmt, + io::{self, Write}, +}; + +pub trait Jsonable { + fn to_json(&self) -> Json; +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Json { + Null, + Bool(bool), + Integer(isize), + UnsignedInteger(usize), + Floating(f64), + String(String), + Array(Vec<Json>), + Object(Vec<(String, Json)>), +} + +impl Json { + /// Convenience method for constructing a JSON array. + pub fn array<I, T>(iter: I) -> Self + where + I: IntoIterator<Item = T>, + T: Into<Json>, + { + Json::Array(iter.into_iter().map(Into::into).collect()) + } + + pub fn empty_array() -> Self { + Json::Array(Vec::new()) + } + + pub fn object() -> JsonObjectBuilder { + JsonObjectBuilder::new() + } + + pub fn marshal<W: Write>(&self, writer: &mut W) -> JsonResult<()> { + match self { + Json::Null => writer.write_all(b"null"), + Json::Bool(b) => writer.write_all(if *b { b"true" } else { b"false" }), + Json::Integer(i) => write!(writer, "{i}"), + Json::UnsignedInteger(u) => write!(writer, "{u}"), + Json::Floating(f) => write!(writer, "{f}"), + Json::String(s) => return Self::write_str(writer, s), + Json::Array(jsons) => return Self::write_array(writer, jsons), + Json::Object(map) => return Self::write_object(writer, map), + }?; + Ok(()) + } + + pub fn write_str<W: Write>(writer: &mut W, s: &str) -> JsonResult<()> { + writer.write_all(b"\"")?; + + for ch in s.chars() { + match ch { + '"' => write!(writer, "\\\"")?, + '\\' => write!(writer, "\\\\")?, + // The following characters are control, but have a canonical representation. + // https://datatracker.ietf.org/doc/html/rfc8259#section-7 + '\n' => write!(writer, "\\n")?, + '\r' => write!(writer, "\\r")?, + '\t' => write!(writer, "\\t")?, + '\x08' => write!(writer, "\\b")?, + '\x0C' => write!(writer, "\\f")?, + ch if ch.is_control() => { + let code_point = ch as u32; + write!(writer, "\\u{code_point:04X}")? + } + _ => write!(writer, "{ch}")?, + }; + } + + writer.write_all(b"\"")?; + Ok(()) + } + + pub fn write_array<W: Write>(writer: &mut W, jsons: &[Json]) -> JsonResult<()> { + writer.write_all(b"[")?; + let mut prefix = ""; + for item in jsons { + write!(writer, "{prefix}")?; + item.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"]")?; + Ok(()) + } + + pub fn write_object<W: Write>(writer: &mut W, pairs: &[(String, Json)]) -> JsonResult<()> { + writer.write_all(b"{")?; + let mut prefix = ""; + for (k, v) in pairs { + // Escape the keys, despite not being `Json::String` objects. + write!(writer, "{prefix}")?; + Self::write_str(writer, k)?; + writer.write_all(b":")?; + v.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"}")?; + Ok(()) + } +} + +impl std::fmt::Display for Json { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let mut buf = Vec::new(); + self.marshal(&mut buf).map_err(|_| std::fmt::Error)?; + let s = String::from_utf8(buf).map_err(|_| std::fmt::Error)?; + write!(f, "{s}") + } +} + +pub struct JsonObjectBuilder { + pairs: Vec<(String, Json)>, +} + +impl JsonObjectBuilder { + pub fn new() -> Self { + Self { pairs: Vec::new() } + } + + pub fn insert<K, V>(mut self, key: K, value: V) -> Self + where + K: Into<String>, + V: Into<Json>, + { + self.pairs.push((key.into(), value.into())); + self + } + + pub fn build(self) -> Json { + Json::Object(self.pairs) + } +} + +impl From<&str> for Json { + fn from(s: &str) -> Json { + Json::String(s.to_string()) + } +} + +impl From<String> for Json { + fn from(s: String) -> Json { + Json::String(s) + } +} + +impl From<i32> for Json { + fn from(i: i32) -> Json { + Json::Integer(i as isize) + } +} + +impl From<i64> for Json { + fn from(i: i64) -> Json { + Json::Integer(i as isize) + } +} + +impl From<u32> for Json { + fn from(u: u32) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<u64> for Json { + fn from(u: u64) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<usize> for Json { + fn from(u: usize) -> Json { + Json::UnsignedInteger(u) + } +} + +impl From<bool> for Json { + fn from(b: bool) -> Json { + Json::Bool(b) + } +} + +impl TryFrom<f64> for Json { + type Error = JsonError; + fn try_from(f: f64) -> Result<Self, Self::Error> { + if f.is_finite() { + Ok(Json::Floating(f)) + } else { + Err(JsonError::FloatError(f)) + } + } +} + +impl<T: Into<Json>> From<Vec<T>> for Json { + fn from(v: Vec<T>) -> Json { + Json::Array(v.into_iter().map(|item| item.into()).collect()) + } +} + +/// Convenience type for a result in JSON serialization. +pub type JsonResult<W> = std::result::Result<W, JsonError>; + +#[derive(Debug)] +pub enum JsonError { + /// Wrapper for a standard `io::Error`. + IoError(io::Error), + /// On attempting to serialize an invalid `f32` or `f64`. + /// Stores invalid values as 64 bit float. + FloatError(f64), +} + +impl From<io::Error> for JsonError { + fn from(err: io::Error) -> Self { + JsonError::IoError(err) + } +} + +impl fmt::Display for JsonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonError::FloatError(v) => write!(f, "Cannot serialize float {v}"), + JsonError::IoError(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for JsonError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + JsonError::IoError(e) => Some(e), + JsonError::FloatError(_) => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_snapshot; + + fn marshal_to_string(json: &Json) -> String { + let mut buf = Vec::new(); + json.marshal(&mut buf).unwrap(); + String::from_utf8(buf).unwrap() + } + + #[test] + fn test_null() { + let json = Json::Null; + assert_snapshot!(marshal_to_string(&json), @"null"); + } + + #[test] + fn test_bool() { + let json: Json = true.into(); + assert_snapshot!(marshal_to_string(&json), @"true"); + let json: Json = false.into(); + assert_snapshot!(marshal_to_string(&json), @"false"); + } + + #[test] + fn test_integer_positive() { + let json: Json = 42.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_integer_negative() { + let json: Json = (-123).into(); + assert_snapshot!(marshal_to_string(&json), @"-123"); + } + + #[test] + fn test_integer_zero() { + let json: Json = 0.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_floating() { + let json = 2.14159.try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"2.14159"); + } + + #[test] + fn test_floating_negative() { + let json = (-2.5).try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"-2.5"); + } + + #[test] + fn test_floating_error() { + let json: Result<Json, JsonError> = f64::NAN.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::NEG_INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + } + + #[test] + fn test_string_simple() { + let json: Json = "hello".into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello""#); + } + + #[test] + fn test_string_empty() { + let json: Json = "".into(); + assert_snapshot!(marshal_to_string(&json), @r#""""#); + } + + #[test] + fn test_string_with_quotes() { + let json: Json = r#"hello "world""#.into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello \"world\"""#); + } + + #[test] + fn test_string_with_backslash() { + let json: Json = r"path\to\file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path\\to\\file""#); + } + + #[test] + fn test_string_with_slash() { + let json: Json = "path/to/file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path/to/file""#); + } + + #[test] + fn test_string_with_newline() { + let json: Json = "line1\nline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\nline2""#); + } + + #[test] + fn test_string_with_carriage_return() { + let json: Json = "line1\rline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\rline2""#); + } + + #[test] + fn test_string_with_tab() { + let json: Json = "col1\tcol2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""col1\tcol2""#); + } + + #[test] + fn test_string_with_backspace() { + let json: Json = "text\x08back".into(); + assert_snapshot!(marshal_to_string(&json), @r#""text\bback""#); + } + + #[test] + fn test_string_with_form_feed() { + let json: Json = "page\x0Cnew".into(); + assert_snapshot!(marshal_to_string(&json), @r#""page\fnew""#); + } + + #[test] + fn test_string_with_control_chars() { + let json: Json = "test\x01\x02\x03".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test\u0001\u0002\u0003""#); + } + + #[test] + fn test_string_with_all_escapes() { + let json: Json = "\"\\/\n\r\t\x08\x0C".into(); + assert_snapshot!(marshal_to_string(&json), @r#""\"\\/\n\r\t\b\f""#); + } + + #[test] + fn test_array_empty() { + let json: Json = Vec::<i32>::new().into(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_array_single_element() { + let json: Json = vec![42].into(); + assert_snapshot!(marshal_to_string(&json), @"[42]"); + } + + #[test] + fn test_array_multiple_elements() { + let json: Json = vec![1, 2, 3].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_array_mixed_types() { + let json = Json::Array(vec![ + Json::Null, + true.into(), + 42.into(), + 3.134.try_into().unwrap(), + "hello".into(), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"[null, true, 42, 3.134, "hello"]"#); + } + + #[test] + fn test_array_nested() { + let json = Json::Array(vec![1.into(), vec![2, 3].into(), 4.into()]); + assert_snapshot!(marshal_to_string(&json), @"[1, [2, 3], 4]"); + } + + #[test] + fn test_object_empty() { + let json = Json::Object(vec![]); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_single_field() { + let json = Json::Object(vec![("key".to_string(), "value".into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_multiple_fields() { + let json = Json::Object(vec![ + ("name".to_string(), "Alice".into()), + ("age".to_string(), 30.into()), + ("active".to_string(), true.into()), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_with_escaped_key() { + let json = Json::Object(vec![("key\nwith\nnewlines".to_string(), 42.into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key\nwith\nnewlines":42}"#); + } + + #[test] + fn test_object_nested() { + let inner = Json::Object(vec![("inner_key".to_string(), "inner_value".into())]); + let json = Json::Object(vec![("outer_key".to_string(), inner)]); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_from_str() { + let json: Json = "test string".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test string""#); + } + + #[test] + fn test_from_i32() { + let json: Json = 42i32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_i64() { + let json: Json = 9223372036854775807i64.into(); + assert_snapshot!(marshal_to_string(&json), @"9223372036854775807"); + } + + #[test] + fn test_from_u32() { + let json: Json = 42u32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_u64() { + let json: Json = 18446744073709551615u64.into(); + assert_snapshot!(marshal_to_string(&json), @"18446744073709551615"); + } + + #[test] + fn test_unsigned_integer_zero() { + let json: Json = 0u64.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_from_bool() { + let json_true: Json = true.into(); + let json_false: Json = false.into(); + assert_snapshot!(marshal_to_string(&json_true), @"true"); + assert_snapshot!(marshal_to_string(&json_false), @"false"); + } + + #[test] + fn test_from_vec() { + let json: Json = vec![1i32, 2i32, 3i32].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_from_vec_strings() { + let json: Json = vec!["a", "b", "c"].into(); + assert_snapshot!(marshal_to_string(&json), @r#"["a", "b", "c"]"#); + } + + #[test] + fn test_complex_nested_structure() { + let settings = Json::Object(vec![ + ("notifications".to_string(), true.into()), + ("theme".to_string(), "dark".into()), + ]); + + let json = Json::Object(vec![ + ("id".to_string(), 1.into()), + ("name".to_string(), "Alice".into()), + ("tags".to_string(), vec!["admin", "user"].into()), + ("settings".to_string(), settings), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"id":1, "name":"Alice", "tags":["admin", "user"], "settings":{"notifications":true, "theme":"dark"}}"#); + } + + #[test] + fn test_deeply_nested_arrays() { + let json = Json::Array(vec![ + Json::Array(vec![vec![1, 2].into(), 3.into()]), + 4.into(), + ]); + assert_snapshot!(marshal_to_string(&json), @"[[[1, 2], 3], 4]"); + } + + #[test] + fn test_unicode_string() { + let json: Json = "兵马俑".into(); + assert_snapshot!(marshal_to_string(&json), @r#""兵马俑""#); + } + + #[test] + fn test_json_array_convenience() { + let json = Json::array(vec![1, 2, 3]); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_json_array_from_iterator() { + let json = Json::array([1, 2, 3].iter().map(|&x| x * 2)); + assert_snapshot!(marshal_to_string(&json), @"[2, 4, 6]"); + } + + #[test] + fn test_json_empty_array() { + let json = Json::empty_array(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_object_builder_empty() { + let json = Json::object().build(); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_builder_single_field() { + let json = Json::object().insert("key", "value").build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_builder_multiple_fields() { + let json = Json::object() + .insert("name", "Alice") + .insert("age", 30) + .insert("active", true) + .build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_builder_with_nested_objects() { + let inner = Json::object().insert("inner_key", "inner_value").build(); + let json = Json::object().insert("outer_key", inner).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_object_builder_with_array() { + let json = Json::object().insert("items", vec![1, 2, 3]).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"items":[1, 2, 3]}"#); + } + + #[test] + fn test_display_trait() { + let json = Json::object() + .insert("name", "Bob") + .insert("count", 42) + .build(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#"{"name":"Bob", "count":42}"#); + } + + #[test] + fn test_display_trait_array() { + let json: Json = vec![1, 2, 3].into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @"[1, 2, 3]"); + } + + #[test] + fn test_display_trait_string() { + let json: Json = "test".into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#""test""#); + } + + #[test] + fn test_from_usize() { + let json: Json = 123usize.into(); + assert_snapshot!(marshal_to_string(&json), @"123"); + } + + #[test] + fn test_from_usize_large() { + let json: Json = usize::MAX.into(); + let expected = format!("{}", usize::MAX); + assert_eq!(marshal_to_string(&json), expected); + } + + #[test] + fn test_json_error_float_display() { + let err = JsonError::FloatError(f64::NAN); + let display_output = format!("{}", err); + assert!(display_output.contains("Cannot serialize float")); + assert!(display_output.contains("NaN")); + } + + #[test] + fn test_json_error_float_display_infinity() { + let err = JsonError::FloatError(f64::INFINITY); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"Cannot serialize float inf"); + } + + #[test] + fn test_json_error_io_display() { + let io_err = io::Error::new(io::ErrorKind::WriteZero, "write error"); + let err = JsonError::IoError(io_err); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"write error"); + } + + #[test] + fn test_io_error_during_marshal() { + struct FailingWriter; + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> io::Result<usize> { + Err(io::Error::other("simulated write failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let json: Json = "test".into(); + let mut writer = FailingWriter; + let result = json.marshal(&mut writer); + assert!(result.is_err()); + assert!(matches!(result, Err(JsonError::IoError(_)))); + } + + #[test] + fn test_clone_json() { + let json1: Json = vec![1, 2, 3].into(); + let json2 = json1.clone(); + assert_eq!(json1, json2); + } + + #[test] + fn test_debug_json() { + let json: Json = "test".into(); + let debug_output = format!("{:?}", json); + assert!(debug_output.contains("String")); + assert!(debug_output.contains("test")); + } + + #[test] + fn test_partial_eq_json() { + let json1: Json = 42.into(); + let json2: Json = 42.into(); + let json3: Json = 43.into(); + assert_eq!(json1, json2); + assert_ne!(json1, json3); + } +} diff --git a/zjit/src/lib.rs b/zjit/src/lib.rs index 8ccb6ae4c1..a79989c912 100644 --- a/zjit/src/lib.rs +++ b/zjit/src/lib.rs @@ -1,15 +1,21 @@ #![allow(dead_code)] #![allow(static_mut_refs)] +#![allow(clippy::enum_variant_names)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::needless_bool)] + // Add std docs to cargo doc. #[doc(inline)] pub use std; mod state; +mod distribution; mod cruby; mod cruby_methods; mod hir; mod hir_type; +mod hir_effect; mod codegen; mod stats; mod cast; @@ -21,3 +27,8 @@ mod disasm; mod options; mod profile; mod invariants; +mod bitset; +mod gc; +mod payload; +mod json; +mod ttycolors; diff --git a/zjit/src/options.rs b/zjit/src/options.rs index 476640d11f..40b49146b7 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -1,59 +1,163 @@ +//! Configurable options for ZJIT. + use std::{ffi::{CStr, CString}, ptr::null}; use std::os::raw::{c_char, c_int, c_uint}; +use crate::cruby::*; +use crate::stats::Counter; +use std::collections::HashSet; + +/// Default --zjit-num-profiles +const DEFAULT_NUM_PROFILES: NumProfiles = 5; +pub type NumProfiles = u32; + +/// Default --zjit-call-threshold. This should be large enough to avoid compiling +/// warmup code, but small enough to perform well on micro-benchmarks. +pub const DEFAULT_CALL_THRESHOLD: CallThreshold = 30; +pub type CallThreshold = u64; /// Number of calls to start profiling YARV instructions. /// They are profiled `rb_zjit_call_threshold - rb_zjit_profile_threshold` times, /// which is equal to --zjit-num-profiles. #[unsafe(no_mangle)] #[allow(non_upper_case_globals)] -pub static mut rb_zjit_profile_threshold: u64 = 1; +pub static mut rb_zjit_profile_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD - DEFAULT_NUM_PROFILES as CallThreshold; /// Number of calls to compile ISEQ with ZJIT at jit_compile() in vm.c. /// --zjit-call-threshold=1 compiles on first execution without profiling information. #[unsafe(no_mangle)] #[allow(non_upper_case_globals)] -pub static mut rb_zjit_call_threshold: u64 = 2; +pub static mut rb_zjit_call_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD; -#[derive(Clone, Copy, Debug)] +/// ZJIT command-line options. This is set before rb_zjit_init() sets +/// ZJITState so that we can query some options while loading builtins. +pub static mut OPTIONS: Option<Options> = None; + +#[derive(Clone, Debug)] pub struct Options { + /// Hard limit of the executable memory block to allocate in bytes. + /// Note that the command line argument is expressed in MiB and not bytes. + pub exec_mem_bytes: usize, + + /// Hard limit of ZJIT's total memory usage. + /// Note that the command line argument is expressed in MiB and not bytes. + pub mem_bytes: usize, + /// Number of times YARV instructions should be profiled. - pub num_profiles: u64, + pub num_profiles: NumProfiles, + + /// Enable ZJIT statistics + pub stats: bool, + + /// Print stats on exit (when stats is also true) + pub print_stats: bool, + + /// Print stats to file on exit (when stats is also true) + pub print_stats_file: Option<std::path::PathBuf>, /// Enable debug logging pub debug: bool, + // Whether to enable JIT at boot. This option prevents other + // ZJIT tuning options from enabling ZJIT at boot. + pub disable: bool, + + /// Turn off the HIR optimizer + pub disable_hir_opt: bool, + /// Dump initial High-level IR before optimization pub dump_hir_init: Option<DumpHIR>, /// Dump High-level IR after optimization, right before codegen. pub dump_hir_opt: Option<DumpHIR>, + /// Dump High-level IR to the given file in Graphviz format after optimization + pub dump_hir_graphviz: Option<std::path::PathBuf>, + + /// Dump High-level IR in Iongraph JSON format after optimization to /tmp/zjit-iongraph-{$PID} + pub dump_hir_iongraph: bool, + /// Dump low-level IR - pub dump_lir: bool, + pub dump_lir: Option<HashSet<DumpLIR>>, /// Dump all compiled machine code. pub dump_disasm: bool, + + /// Trace and write side exit source maps to /tmp for stackprof. + pub trace_side_exits: Option<TraceExits>, + + /// Frequency of tracing side exits. + pub trace_side_exits_sample_interval: usize, + + /// Dump code map to /tmp for performance profilers. + pub perf: bool, + + /// List of ISEQs that can be compiled, identified by their iseq_get_location() + pub allowed_iseqs: Option<HashSet<String>>, + + /// Path to a file where compiled ISEQs will be saved. + pub log_compiled_iseqs: Option<std::path::PathBuf>, } -/// Return an Options with default values -pub fn init_options() -> Options { - Options { - num_profiles: 1, - debug: false, - dump_hir_init: None, - dump_hir_opt: None, - dump_lir: false, - dump_disasm: false, +impl Default for Options { + fn default() -> Self { + Options { + exec_mem_bytes: 64 * 1024 * 1024, + mem_bytes: 128 * 1024 * 1024, + num_profiles: DEFAULT_NUM_PROFILES, + stats: false, + print_stats: false, + print_stats_file: None, + debug: false, + disable: false, + disable_hir_opt: false, + dump_hir_init: None, + dump_hir_opt: None, + dump_hir_graphviz: None, + dump_hir_iongraph: false, + dump_lir: None, + dump_disasm: false, + trace_side_exits: None, + trace_side_exits_sample_interval: 0, + perf: false, + allowed_iseqs: None, + log_compiled_iseqs: None, + } } } /// `ruby --help` descriptions for user-facing options. Do not add options for ZJIT developers. -/// Note that --help allows only 80 chars per line, including indentation. 80-char limit --> | -pub const ZJIT_OPTIONS: &'static [(&str, &str)] = &[ - ("--zjit-call-threshold=num", "Number of calls to trigger JIT (default: 2)."), - ("--zjit-num-profiles=num", "Number of profiled calls before JIT (default: 1)."), +/// Note that --help allows only 80 chars per line, including indentation, and it also puts the +/// description in a separate line if the option name is too long. 80-char limit --> | (any character beyond this `|` column fails the test) +pub const ZJIT_OPTIONS: &[(&str, &str)] = &[ + ("--zjit-mem-size=num", + "Max amount of memory that ZJIT can use in MiB (default: 128)."), + ("--zjit-call-threshold=num", + "Number of calls to trigger JIT (default: 30)."), + ("--zjit-num-profiles=num", + "Number of profiled calls before JIT (default: 5)."), + ("--zjit-stats-quiet", + "Collect ZJIT stats and suppress output."), + ("--zjit-stats[=file]", + "Collect ZJIT stats (=file to write to a file)."), + ("--zjit-disable", + "Disable ZJIT for lazily enabling it with RubyVM::ZJIT.enable."), + ("--zjit-perf", "Dump ISEQ symbols into /tmp/perf-{}.map for Linux perf."), + ("--zjit-log-compiled-iseqs=path", + "Log compiled ISEQs to the file. The file will be truncated."), + ("--zjit-trace-exits[=counter]", + "Record source on side-exit. `Counter` picks specific counter."), + ("--zjit-trace-exits-sample-rate=num", + "Frequency at which to record side exits. Must be `usize`.") ]; +#[derive(Copy, Clone, Debug)] +pub enum TraceExits { + // Trace all exits + All, + // Trace exits for a specific `Counter` + Counter(Counter), +} + #[derive(Clone, Copy, Debug)] pub enum DumpHIR { // Dump High-level IR without Snapshot @@ -64,39 +168,101 @@ pub enum DumpHIR { Debug, } +/// --zjit-dump-lir values. Using snake_case to stringify the exact filter value. +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub enum DumpLIR { + /// Dump the initial LIR + init, + /// Dump LIR after {arch}_split + split, + /// Dump LIR after alloc_regs + alloc_regs, + /// Dump LIR after compile_exits + compile_exits, + /// Dump LIR after {arch}_scratch_split + scratch_split, +} + +/// All compiler stages for --zjit-dump-lir=all. +const DUMP_LIR_ALL: &[DumpLIR] = &[ + DumpLIR::init, + DumpLIR::split, + DumpLIR::alloc_regs, + DumpLIR::compile_exits, + DumpLIR::scratch_split, +]; + +/// Maximum value for --zjit-mem-size/--zjit-exec-mem-size in MiB. +/// We set 1TiB just to avoid overflow. We could make it smaller. +const MAX_MEM_MIB: usize = 1024 * 1024; + +/// Macro to dump LIR if --zjit-dump-lir is specified +macro_rules! asm_dump { + ($asm:expr, $target:ident) => { + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::$target) { + println!("LIR {}:\n{}", stringify!($target), $asm); + } + } + }; +} +pub(crate) use asm_dump; + /// Macro to get an option value by name macro_rules! get_option { // Unsafe is ok here because options are initialized // once before any Ruby code executes ($option_name:ident) => { - { - use crate::state::ZJITState; - ZJITState::get_options().$option_name - } + unsafe { crate::options::OPTIONS.as_ref() }.unwrap().$option_name }; } pub(crate) use get_option; -/// Allocate Options on the heap, initialize it, and return the address of it. -/// The return value will be modified by rb_zjit_parse_option() and then -/// passed to rb_zjit_init() for initialization. +/// Set default values to ZJIT options. Setting Some to OPTIONS will make `#with_jit` +/// enable the JIT hook while not enabling compilation yet. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_init_options() -> *const u8 { - let options = init_options(); - Box::into_raw(Box::new(options)) as *const u8 +pub extern "C" fn rb_zjit_prepare_options() { + // rb_zjit_prepare_options() could be called for feature flags or $RUBY_ZJIT_ENABLE + // after rb_zjit_parse_option() is called, so we need to handle the already-initialized case. + if unsafe { OPTIONS.is_none() } { + unsafe { OPTIONS = Some(Options::default()); } + } } /// Parse a --zjit* command-line flag #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_parse_option(options: *const u8, str_ptr: *const c_char) -> bool { - let options = unsafe { &mut *(options as *mut Options) }; - parse_option(options, str_ptr).is_some() +pub extern "C" fn rb_zjit_parse_option(str_ptr: *const c_char) -> bool { + parse_option(str_ptr).is_some() +} + +fn parse_jit_list(path_like: &str) -> HashSet<String> { + // Read lines from the file + let mut result = HashSet::new(); + if let Ok(lines) = std::fs::read_to_string(path_like) { + for line in lines.lines() { + let trimmed = line.trim(); + if !trimmed.is_empty() { + result.insert(trimmed.to_string()); + } + } + } else { + eprintln!("Failed to read JIT list from '{path_like}'"); + } + eprintln!("JIT list:"); + for item in &result { + eprintln!(" {item}"); + } + result } /// Expected to receive what comes after the third dash in "--zjit-*". /// Empty string means user passed only "--zjit". C code rejects when /// they pass exact "--zjit-". -fn parse_option(options: &mut Options, str_ptr: *const std::os::raw::c_char) -> Option<()> { +fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { + rb_zjit_prepare_options(); + let options = unsafe { OPTIONS.as_mut().unwrap() }; + let c_str: &CStr = unsafe { CStr::from_ptr(str_ptr) }; let opt_str: &str = c_str.to_str().ok()?; @@ -112,10 +278,26 @@ fn parse_option(options: &mut Options, str_ptr: *const std::os::raw::c_char) -> match (opt_name, opt_val) { ("", "") => {}, // Simply --zjit + ("mem-size", _) => match opt_val.parse::<usize>() { + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { + // Convert from MiB to bytes internally for convenience + options.mem_bytes = n * 1024 * 1024; + } + _ => return None, + }, + + ("exec-mem-size", _) => match opt_val.parse::<usize>() { + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { + // Convert from MiB to bytes internally for convenience + options.exec_mem_bytes = n * 1024 * 1024; + } + _ => return None, + }, + ("call-threshold", _) => match opt_val.parse() { Ok(n) => { unsafe { rb_zjit_call_threshold = n; } - update_profile_threshold(options); + update_profile_threshold(); }, Err(_) => return None, }, @@ -123,13 +305,57 @@ fn parse_option(options: &mut Options, str_ptr: *const std::os::raw::c_char) -> ("num-profiles", _) => match opt_val.parse() { Ok(n) => { options.num_profiles = n; - update_profile_threshold(options); + update_profile_threshold(); }, Err(_) => return None, }, + + ("stats-quiet", _) => { + options.stats = true; + options.print_stats = false; + } + + ("stats", "") => { + options.stats = true; + options.print_stats = true; + } + ("stats", path) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path) + .map_err(|e| eprintln!("Failed to open file '{}': {}", path, e)) + .ok(); + let canonical_path = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.stats = true; + options.print_stats_file = Some(canonical_path); + } + + ("trace-exits", exits) => { + options.trace_side_exits = match exits { + "" => Some(TraceExits::All), + name => Some(Counter::get(name).map(TraceExits::Counter)?), + } + } + + ("trace-exits-sample-rate", sample_interval) => { + // If not already set, then set it to `TraceExits::All` by default. + if options.trace_side_exits.is_none() { + options.trace_side_exits = Some(TraceExits::All); + } + // `sample_interval ` must provide a string that can be validly parsed to a `usize`. + options.trace_side_exits_sample_interval = sample_interval.parse::<usize>().ok()?; + } + ("debug", "") => options.debug = true, + ("disable", "") => options.disable = true, + + ("disable-hir-opt", "") => options.disable_hir_opt = true, + // --zjit-dump-hir dumps the actual input to the codegen, which is currently the same as --zjit-dump-hir-opt. ("dump-hir" | "dump-hir-opt", "") => options.dump_hir_opt = Some(DumpHIR::WithoutSnapshot), ("dump-hir" | "dump-hir-opt", "all") => options.dump_hir_opt = Some(DumpHIR::All), @@ -139,10 +365,68 @@ fn parse_option(options: &mut Options, str_ptr: *const std::os::raw::c_char) -> ("dump-hir-init", "all") => options.dump_hir_init = Some(DumpHIR::All), ("dump-hir-init", "debug") => options.dump_hir_init = Some(DumpHIR::Debug), - ("dump-lir", "") => options.dump_lir = true, + ("dump-hir-graphviz", "") => options.dump_hir_graphviz = Some("/dev/stderr".into()), + ("dump-hir-graphviz", _) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(opt_val) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) + .ok(); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.dump_hir_graphviz = Some(opt_val); + } + + ("dump-hir-iongraph", "") => options.dump_hir_iongraph = true, + + ("dump-lir", "") => options.dump_lir = Some(HashSet::from([DumpLIR::init])), + ("dump-lir", filters) => { + let mut dump_lirs = HashSet::new(); + for filter in filters.split(',') { + let dump_lir = match filter { + "all" => { + for &dump_lir in DUMP_LIR_ALL { + dump_lirs.insert(dump_lir); + } + continue; + } + "init" => DumpLIR::init, + "split" => DumpLIR::split, + "alloc_regs" => DumpLIR::alloc_regs, + "compile_exits" => DumpLIR::compile_exits, + "scratch_split" => DumpLIR::scratch_split, + _ => { + let valid_options = DUMP_LIR_ALL.iter().map(|opt| format!("{opt:?}")).collect::<Vec<_>>().join(", "); + eprintln!("invalid --zjit-dump-lir option: '{filter}'"); + eprintln!("valid --zjit-dump-lir options: all, {valid_options}"); + return None; + } + }; + dump_lirs.insert(dump_lir); + } + options.dump_lir = Some(dump_lirs); + } ("dump-disasm", "") => options.dump_disasm = true, + ("perf", "") => options.perf = true, + + ("allowed-iseqs", _) if !opt_val.is_empty() => options.allowed_iseqs = Some(parse_jit_list(opt_val)), + ("log-compiled-iseqs", _) if !opt_val.is_empty() => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(opt_val) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) + .ok(); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.log_compiled_iseqs = Some(opt_val); + } + _ => return None, // Option name not recognized } @@ -151,18 +435,25 @@ fn parse_option(options: &mut Options, str_ptr: *const std::os::raw::c_char) -> } /// Update rb_zjit_profile_threshold based on rb_zjit_call_threshold and options.num_profiles -fn update_profile_threshold(options: &Options) { - unsafe { - if rb_zjit_call_threshold == 1 { - // If --zjit-call-threshold=1, never rewrite ISEQs to profile instructions. - rb_zjit_profile_threshold = 0; - } else { - // Otherwise, profile instructions at least once. - rb_zjit_profile_threshold = rb_zjit_call_threshold.saturating_sub(options.num_profiles).max(1); - } +fn update_profile_threshold() { + if unsafe { rb_zjit_call_threshold == 1 } { + // If --zjit-call-threshold=1, never rewrite ISEQs to profile instructions. + unsafe { rb_zjit_profile_threshold = 0; } + } else { + // Otherwise, profile instructions at least once. + let num_profiles = get_option!(num_profiles); + unsafe { rb_zjit_profile_threshold = rb_zjit_call_threshold.saturating_sub(num_profiles.into()).max(1) }; } } +/// Update --zjit-call-threshold for testing +#[cfg(test)] +pub fn set_call_threshold(call_threshold: CallThreshold) { + unsafe { rb_zjit_call_threshold = call_threshold; } + rb_zjit_prepare_options(); + update_profile_threshold(); +} + /// Print YJIT options for `ruby --help`. `width` is width of option parts, and /// `columns` is indent width of descriptions. #[unsafe(no_mangle)] @@ -187,3 +478,46 @@ macro_rules! debug { }; } pub(crate) use debug; + +/// Return true if ZJIT should be enabled at boot. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_option_enable() -> bool { + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| !opts.disable) { + true + } else { + false + } +} + +/// Return Qtrue if --zjit-stats has been specified. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_stats_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats) { + Qtrue + } else { + Qfalse + } +} + +/// Return Qtrue if stats should be printed at exit. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_print_stats_p(_ec: EcPtr, _self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats && opts.print_stats) { + Qtrue + } else { + Qfalse + } +} + +/// Return path if stats should be printed at exit to a specified file, else Qnil. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_get_stats_file_path_p(_ec: EcPtr, _self: VALUE) -> VALUE { + if let Some(opts) = unsafe { OPTIONS.as_ref() } { + if let Some(ref path) = opts.print_stats_file { + return rust_str_to_ruby(path.as_os_str().to_str().unwrap()); + } + } + Qnil +} diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs new file mode 100644 index 0000000000..8540d5e35c --- /dev/null +++ b/zjit/src/payload.rs @@ -0,0 +1,116 @@ +use std::ffi::c_void; +use std::ptr::NonNull; +use crate::codegen::IseqCallRef; +use crate::stats::CompileError; +use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; + +/// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. +#[derive(Debug)] +pub struct IseqPayload { + /// Type information of YARV instruction operands + pub profile: IseqProfile, + /// JIT code versions. Different versions should have different assumptions. + pub versions: Vec<IseqVersionRef>, +} + +impl IseqPayload { + fn new(iseq_size: u32) -> Self { + Self { + profile: IseqProfile::new(iseq_size), + versions: vec![], + } + } +} + +/// JIT code version. When the same ISEQ is compiled with a different assumption, a new version is created. +#[derive(Debug)] +pub struct IseqVersion { + /// ISEQ pointer. Stored here to minimize the size of PatchPoint. + pub iseq: IseqPtr, + + /// Compilation status of the ISEQ. It has the JIT code address of the first block if Compiled. + pub status: IseqStatus, + + /// GC offsets of the JIT code. These are the addresses of objects that need to be marked. + pub gc_offsets: Vec<CodePtr>, + + /// JIT-to-JIT calls from the ISEQ. The IseqPayload's ISEQ is the caller of it. + pub outgoing: Vec<IseqCallRef>, + + /// JIT-to-JIT calls to the ISEQ. The IseqPayload's ISEQ is the callee of it. + pub incoming: Vec<IseqCallRef>, +} + +/// We use a raw pointer instead of Rc to save space for refcount +pub type IseqVersionRef = NonNull<IseqVersion>; + +impl IseqVersion { + /// Allocate a new IseqVersion to be compiled + pub fn new(iseq: IseqPtr) -> IseqVersionRef { + let version = Self { + iseq, + status: IseqStatus::NotCompiled, + gc_offsets: vec![], + outgoing: vec![], + incoming: vec![], + }; + let version_ptr = Box::into_raw(Box::new(version)); + NonNull::new(version_ptr).expect("no null from Box") + } +} + +/// Set of CodePtrs for an ISEQ +#[derive(Clone, Debug, PartialEq)] +pub struct IseqCodePtrs { + /// Entry for the interpreter + pub start_ptr: CodePtr, + /// Entries for JIT-to-JIT calls + pub jit_entry_ptrs: Vec<CodePtr>, +} + +#[derive(Debug, PartialEq)] +pub enum IseqStatus { + Compiled(IseqCodePtrs), + CantCompile(CompileError), + NotCompiled, + Invalidated, +} + +/// Get a pointer to the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload_ptr(iseq: IseqPtr) -> *mut IseqPayload { + type VoidPtr = *mut c_void; + + unsafe { + let payload = rb_iseq_get_zjit_payload(iseq); + if payload.is_null() { + // Allocate a new payload with Box and transfer ownership to the GC. + // We drop the payload with Box::from_raw when the GC frees the ISEQ and calls us. + // NOTE(alan): Sometimes we read from an ISEQ without ever writing to it. + // We allocate in those cases anyways. + let iseq_size = get_iseq_encoded_size(iseq); + let new_payload = IseqPayload::new(iseq_size); + let new_payload = Box::into_raw(Box::new(new_payload)); + rb_iseq_set_zjit_payload(iseq, new_payload as VoidPtr); + + new_payload + } else { + payload as *mut IseqPayload + } + } +} + +/// Get the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { + let payload_non_null = get_or_create_iseq_payload_ptr(iseq); + payload_ptr_as_mut(payload_non_null) +} + +/// Convert an IseqPayload pointer to a mutable reference. Only one reference +/// should be kept at a time. +pub fn payload_ptr_as_mut(payload_ptr: *mut IseqPayload) -> &'static mut IseqPayload { + // SAFETY: we should have the VM lock and all other Ruby threads should be asleep. So we have + // exclusive mutable access. + // Hmm, nothing seems to stop calling this on the same + // iseq twice, though, which violates aliasing rules. + unsafe { payload_ptr.as_mut() }.unwrap() +} diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 5b88b08b14..7a584afd6f 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -1,10 +1,13 @@ +//! Profiler for runtime information. + // We use the YARV bytecode constants which have a CRuby-style name #![allow(non_upper_case_globals)] -use core::ffi::c_void; use std::collections::HashMap; - -use crate::{cruby::*, hir_type::{types::{Empty, Fixnum}, Type}, virtualmem::CodePtr}; +use crate::{cruby::*, payload::get_or_create_iseq_payload, options::{get_option, NumProfiles}}; +use crate::distribution::{Distribution, DistributionSummary}; +use crate::stats::Counter::profile_time_ns; +use crate::stats::with_time_stat; /// Ephemeral state for profiling runtime information struct Profiler { @@ -37,116 +40,395 @@ impl Profiler { *(sp.offset(-1 - n)) } } + + fn peek_at_self(&self) -> VALUE { + unsafe { rb_get_cfp_self(self.cfp) } + } + + fn peek_at_block_handler(&self) -> VALUE { + unsafe { rb_vm_get_untagged_block_handler(self.cfp) } + } } /// API called from zjit_* instruction. opcode is the bare (non-zjit_*) instruction. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_profile_insn(opcode: ruby_vminsn_type, ec: EcPtr) { +pub extern "C" fn rb_zjit_profile_insn(bare_opcode: u32, ec: EcPtr) { with_vm_lock(src_loc!(), || { - let mut profiler = Profiler::new(ec); - profile_insn(&mut profiler, opcode); + with_time_stat(profile_time_ns, || profile_insn(bare_opcode as ruby_vminsn_type, ec)); }); } /// Profile a YARV instruction -fn profile_insn(profiler: &mut Profiler, opcode: ruby_vminsn_type) { - match opcode { - YARVINSN_opt_plus => profile_operands(profiler, 2), - YARVINSN_opt_minus => profile_operands(profiler, 2), - YARVINSN_opt_mult => profile_operands(profiler, 2), - YARVINSN_opt_div => profile_operands(profiler, 2), - YARVINSN_opt_mod => profile_operands(profiler, 2), - YARVINSN_opt_eq => profile_operands(profiler, 2), - YARVINSN_opt_neq => profile_operands(profiler, 2), - YARVINSN_opt_lt => profile_operands(profiler, 2), - YARVINSN_opt_le => profile_operands(profiler, 2), - YARVINSN_opt_gt => profile_operands(profiler, 2), - YARVINSN_opt_ge => profile_operands(profiler, 2), - YARVINSN_opt_send_without_block => { +fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) { + let profiler = &mut Profiler::new(ec); + let profile = &mut get_or_create_iseq_payload(profiler.iseq).profile; + match bare_opcode { + YARVINSN_opt_nil_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_plus => profile_operands(profiler, profile, 2), + YARVINSN_opt_minus => profile_operands(profiler, profile, 2), + YARVINSN_opt_mult => profile_operands(profiler, profile, 2), + YARVINSN_opt_div => profile_operands(profiler, profile, 2), + YARVINSN_opt_mod => profile_operands(profiler, profile, 2), + YARVINSN_opt_eq => profile_operands(profiler, profile, 2), + YARVINSN_opt_neq => profile_operands(profiler, profile, 2), + YARVINSN_opt_lt => profile_operands(profiler, profile, 2), + YARVINSN_opt_le => profile_operands(profiler, profile, 2), + YARVINSN_opt_gt => profile_operands(profiler, profile, 2), + YARVINSN_opt_ge => profile_operands(profiler, profile, 2), + YARVINSN_opt_and => profile_operands(profiler, profile, 2), + YARVINSN_opt_or => profile_operands(profiler, profile, 2), + YARVINSN_opt_empty_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_aref => profile_operands(profiler, profile, 2), + YARVINSN_opt_ltlt => profile_operands(profiler, profile, 2), + YARVINSN_opt_aset => profile_operands(profiler, profile, 3), + YARVINSN_opt_not => profile_operands(profiler, profile, 1), + YARVINSN_getinstancevariable => profile_self(profiler, profile), + YARVINSN_setinstancevariable => profile_self(profiler, profile), + YARVINSN_definedivar => profile_self(profiler, profile), + YARVINSN_opt_regexpmatch2 => profile_operands(profiler, profile, 2), + YARVINSN_objtostring => profile_operands(profiler, profile, 1), + YARVINSN_opt_length => profile_operands(profiler, profile, 1), + YARVINSN_opt_size => profile_operands(profiler, profile, 1), + YARVINSN_opt_succ => profile_operands(profiler, profile, 1), + YARVINSN_invokeblock => profile_block_handler(profiler, profile), + YARVINSN_invokesuper => profile_invokesuper(profiler, profile), + YARVINSN_opt_send_without_block | YARVINSN_send => { let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); let argc = unsafe { vm_ci_argc((*cd).ci) }; // Profile all the arguments and self (+1). - profile_operands(profiler, (argc + 1) as usize); + profile_operands(profiler, profile, (argc + 1) as usize); } _ => {} } + + // Once we profile the instruction num_profiles times, we stop profiling it. + profile.num_profiles[profiler.insn_idx] = profile.num_profiles[profiler.insn_idx].saturating_add(1); + if profile.num_profiles[profiler.insn_idx] == get_option!(num_profiles) { + unsafe { rb_zjit_iseq_insn_set(profiler.iseq, profiler.insn_idx as u32, bare_opcode); } + } } +const DISTRIBUTION_SIZE: usize = 4; + +pub type TypeDistribution = Distribution<ProfiledType, DISTRIBUTION_SIZE>; + +pub type TypeDistributionSummary = DistributionSummary<ProfiledType, DISTRIBUTION_SIZE>; + /// Profile the Type of top-`n` stack operands -fn profile_operands(profiler: &mut Profiler, n: usize) { - let payload = get_or_create_iseq_payload(profiler.iseq); - let mut types = if let Some(types) = payload.opnd_types.get(&profiler.insn_idx) { - types.clone() - } else { - vec![Empty; n] - }; +fn profile_operands(profiler: &mut Profiler, profile: &mut IseqProfile, n: usize) { + let types = &mut profile.opnd_types[profiler.insn_idx]; + if types.is_empty() { + types.resize(n, TypeDistribution::new()); + } + + for (i, profile_type) in types.iter_mut().enumerate() { + let obj = profiler.peek_at_stack((n - i - 1) as isize); + // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or + // drop them or something + let ty = ProfiledType::new(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + profile_type.observe(ty); + } +} + +fn profile_self(profiler: &mut Profiler, profile: &mut IseqProfile) { + let types = &mut profile.opnd_types[profiler.insn_idx]; + if types.is_empty() { + types.resize(1, TypeDistribution::new()); + } + let obj = profiler.peek_at_self(); + // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or + // drop them or something + let ty = ProfiledType::new(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + types[0].observe(ty); +} + +fn profile_block_handler(profiler: &mut Profiler, profile: &mut IseqProfile) { + let types = &mut profile.opnd_types[profiler.insn_idx]; + if types.is_empty() { + types.resize(1, TypeDistribution::new()); + } + let obj = profiler.peek_at_block_handler(); + let ty = ProfiledType::object(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + types[0].observe(ty); +} + +fn profile_invokesuper(profiler: &mut Profiler, profile: &mut IseqProfile) { + let cme = unsafe { rb_vm_frame_method_entry(profiler.cfp) }; + let cme_value = VALUE(cme as usize); // CME is a T_IMEMO, which is a VALUE + + match profile.super_cme.get(&profiler.insn_idx) { + None => { + // If `None`, then this is our first time looking at `super` for this instruction. + profile.super_cme.insert(profiler.insn_idx, Some(cme_value)); + }, + Some(Some(existing_cme)) => { + // Check if the stored method entry is the same as the current one. If it isn't, then + // mark the call site as polymorphic. + if *existing_cme != cme_value { + profile.super_cme.insert(profiler.insn_idx, None); + } + } + Some(None) => { + // We've visited this instruction and explicitly stored `None` to mark the call site + // as polymorphic. + } + } + + unsafe { rb_gc_writebarrier(profiler.iseq.into(), cme_value) }; + + let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); + let argc = unsafe { vm_ci_argc((*cd).ci) }; + + // Profile all the arguments and self (+1). + profile_operands(profiler, profile, (argc + 1) as usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Flags(u32); + +impl Flags { + const NONE: u32 = 0; + const IS_IMMEDIATE: u32 = 1 << 0; + /// Object is embedded and the ivar index lands within the object + const IS_EMBEDDED: u32 = 1 << 1; + /// Object is a T_OBJECT + const IS_T_OBJECT: u32 = 1 << 2; + /// Object is a struct with embedded fields + const IS_STRUCT_EMBEDDED: u32 = 1 << 3; + /// Set if the ProfiledType is used for profiling specific objects, not just classes/shapes + const IS_OBJECT_PROFILING: u32 = 1 << 4; + + pub fn none() -> Self { Self(Self::NONE) } + + pub fn immediate() -> Self { Self(Self::IS_IMMEDIATE) } + pub fn is_immediate(self) -> bool { (self.0 & Self::IS_IMMEDIATE) != 0 } + pub fn is_embedded(self) -> bool { (self.0 & Self::IS_EMBEDDED) != 0 } + pub fn is_t_object(self) -> bool { (self.0 & Self::IS_T_OBJECT) != 0 } + pub fn is_struct_embedded(self) -> bool { (self.0 & Self::IS_STRUCT_EMBEDDED) != 0 } + pub fn is_object_profiling(self) -> bool { (self.0 & Self::IS_OBJECT_PROFILING) != 0 } +} + +/// opt_send_without_block/opt_plus/... should store: +/// * the class of the receiver, so we can do method lookup +/// * the shape of the receiver, so we can optimize ivar lookup +/// +/// with those two, pieces of information, we can also determine when an object is an immediate: +/// * Integer + IS_IMMEDIATE == Fixnum +/// * Float + IS_IMMEDIATE == Flonum +/// * Symbol + IS_IMMEDIATE == StaticSymbol +/// * NilClass == Nil +/// * TrueClass == True +/// * FalseClass == False +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProfiledType { + class: VALUE, + shape: ShapeId, + flags: Flags, +} + +impl Default for ProfiledType { + fn default() -> Self { + Self::empty() + } +} + +impl ProfiledType { + /// Profile the object itself + fn object(obj: VALUE) -> Self { + let mut flags = Flags::none(); + flags.0 |= Flags::IS_OBJECT_PROFILING; + Self { class: obj, shape: INVALID_SHAPE_ID, flags } + } + + /// Profile the class and shape of the given object + fn new(obj: VALUE) -> Self { + if obj == Qfalse { + return Self { class: unsafe { rb_cFalseClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qtrue { + return Self { class: unsafe { rb_cTrueClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj == Qnil { + return Self { class: unsafe { rb_cNilClass }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.fixnum_p() { + return Self { class: unsafe { rb_cInteger }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.flonum_p() { + return Self { class: unsafe { rb_cFloat }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + if obj.static_sym_p() { + return Self { class: unsafe { rb_cSymbol }, + shape: INVALID_SHAPE_ID, + flags: Flags::immediate() }; + } + let mut flags = Flags::none(); + if obj.embedded_p() { + flags.0 |= Flags::IS_EMBEDDED; + } + if obj.struct_embedded_p() { + flags.0 |= Flags::IS_STRUCT_EMBEDDED; + } + if unsafe { RB_TYPE_P(obj, RUBY_T_OBJECT) } { + flags.0 |= Flags::IS_T_OBJECT; + } + Self { class: obj.class_of(), shape: obj.shape_id_of(), flags } + } + + pub fn empty() -> Self { + Self { class: VALUE(0), shape: INVALID_SHAPE_ID, flags: Flags::none() } + } + + pub fn is_empty(&self) -> bool { + self.class == VALUE(0) + } + + pub fn class(&self) -> VALUE { + self.class + } - for i in 0..n { - let opnd_type = Type::from_value(profiler.peek_at_stack((n - i - 1) as isize)); - types[i] = types[i].union(opnd_type); + pub fn shape(&self) -> ShapeId { + self.shape } - payload.opnd_types.insert(profiler.insn_idx, types); + pub fn flags(&self) -> Flags { + self.flags + } + + pub fn is_fixnum(&self) -> bool { + self.class == unsafe { rb_cInteger } && self.flags.is_immediate() + } + + pub fn is_string(&self) -> bool { + if self.flags.is_object_profiling() { + panic!("should not call is_string on object-profiled ProfiledType"); + } + // Fast paths for immediates and exact-class + if self.flags.is_immediate() { + return false; + } + + let string = unsafe { rb_cString }; + if self.class == string{ + return true; + } + + self.class.is_subclass_of(string) == ClassRelationship::Subclass + } + + pub fn is_flonum(&self) -> bool { + self.class == unsafe { rb_cFloat } && self.flags.is_immediate() + } + + pub fn is_static_symbol(&self) -> bool { + self.class == unsafe { rb_cSymbol } && self.flags.is_immediate() + } + + pub fn is_nil(&self) -> bool { + self.class == unsafe { rb_cNilClass } && self.flags.is_immediate() + } + + pub fn is_true(&self) -> bool { + self.class == unsafe { rb_cTrueClass } && self.flags.is_immediate() + } + + pub fn is_false(&self) -> bool { + self.class == unsafe { rb_cFalseClass } && self.flags.is_immediate() + } } -/// This is all the data ZJIT stores on an iseq. This will be dynamically allocated by C code -/// C code should pass an &mut IseqPayload to us when calling into ZJIT. -#[derive(Default, Debug)] -pub struct IseqPayload { +#[derive(Debug)] +pub struct IseqProfile { /// Type information of YARV instruction operands, indexed by the instruction index - opnd_types: HashMap<usize, Vec<Type>>, + opnd_types: Vec<Vec<TypeDistribution>>, + + /// Number of profiled executions for each YARV instruction, indexed by the instruction index + num_profiles: Vec<NumProfiles>, - /// JIT code address of the first block - pub start_ptr: Option<CodePtr>, + /// Method entries for `super` calls (stored as VALUE to be GC-safe) + super_cme: HashMap<usize, Option<VALUE>> } -impl IseqPayload { +impl IseqProfile { + pub fn new(iseq_size: u32) -> Self { + Self { + opnd_types: vec![vec![]; iseq_size as usize], + num_profiles: vec![0; iseq_size as usize], + super_cme: HashMap::new(), + } + } + /// Get profiled operand types for a given instruction index - pub fn get_operand_types(&self, insn_idx: usize) -> Option<&[Type]> { - self.opnd_types.get(&insn_idx).map(|types| types.as_slice()) + pub fn get_operand_types(&self, insn_idx: usize) -> Option<&[TypeDistribution]> { + self.opnd_types.get(insn_idx).map(|v| &**v) } - /// Return true if top-two stack operands are Fixnums - pub fn have_two_fixnums(&self, insn_idx: usize) -> bool { - match self.get_operand_types(insn_idx) { - Some([left, right]) => left.is_subtype(Fixnum) && right.is_subtype(Fixnum), - _ => false, + pub fn get_super_method_entry(&self, insn_idx: usize) -> Option<*const rb_callable_method_entry_t> { + self.super_cme.get(&insn_idx) + .and_then(|opt| opt.map(|v| v.0 as *const rb_callable_method_entry_t)) + } + + /// Run a given callback with every object in IseqProfile + pub fn each_object(&self, callback: impl Fn(VALUE)) { + for operands in &self.opnd_types { + for distribution in operands { + for profiled_type in distribution.each_item() { + // If the type is a GC object, call the callback + callback(profiled_type.class); + } + } + } + + for cme_value in self.super_cme.values() { + if let Some(cme) = cme_value { + callback(*cme); + } } } -} -/// Get the payload for an iseq. For safety it's up to the caller to ensure the returned `&mut` -/// upholds aliasing rules and that the argument is a valid iseq. -pub fn get_iseq_payload(iseq: IseqPtr) -> Option<&'static mut IseqPayload> { - let payload = unsafe { rb_iseq_get_zjit_payload(iseq) }; - let payload: *mut IseqPayload = payload.cast(); - unsafe { payload.as_mut() } + /// Run a given callback with a mutable reference to every object in IseqProfile. + pub fn each_object_mut(&mut self, callback: impl Fn(&mut VALUE)) { + for operands in &mut self.opnd_types { + for distribution in operands { + for ref mut profiled_type in distribution.each_item_mut() { + // If the type is a GC object, call the callback + callback(&mut profiled_type.class); + } + } + } + + // Update CME references if they move during compaction. + for cme_value in self.super_cme.values_mut() { + if let Some(cme) = cme_value { + callback(cme); + } + } + } } -/// Get the payload object associated with an iseq. Create one if none exists. -pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { - type VoidPtr = *mut c_void; - - let payload_non_null = unsafe { - let payload = rb_iseq_get_zjit_payload(iseq); - if payload.is_null() { - // Allocate a new payload with Box and transfer ownership to the GC. - // We drop the payload with Box::from_raw when the GC frees the iseq and calls us. - // NOTE(alan): Sometimes we read from an iseq without ever writing to it. - // We allocate in those cases anyways. - let new_payload = IseqPayload::default(); - let new_payload = Box::into_raw(Box::new(new_payload)); - rb_iseq_set_zjit_payload(iseq, new_payload as VoidPtr); - - new_payload - } else { - payload as *mut IseqPayload - } - }; - - // SAFETY: we should have the VM lock and all other Ruby threads should be asleep. So we have - // exclusive mutable access. - // Hmm, nothing seems to stop calling this on the same - // iseq twice, though, which violates aliasing rules. - unsafe { payload_non_null.as_mut() }.unwrap() +#[cfg(test)] +mod tests { + use crate::cruby::*; + + #[test] + fn can_profile_block_handler() { + with_rubyvm(|| eval(" + def foo = yield + foo rescue 0 + foo rescue 0 + ")); + } } diff --git a/zjit/src/state.rs b/zjit/src/state.rs index e846ee6f8d..a807be3f12 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -1,16 +1,24 @@ -use crate::cruby::{self, rb_bug_panic_hook, EcPtr, Qnil, VALUE}; +//! Runtime state of ZJIT. + +use crate::codegen::{gen_entry_trampoline, gen_exit_trampoline, gen_exit_trampoline_with_counter, gen_function_stub_hit_trampoline}; +use crate::cruby::{self, rb_bug_panic_hook, rb_vm_insn_count, src_loc, EcPtr, Qnil, Qtrue, rb_vm_insn_addr2opcode, rb_profile_frames, VALUE, VM_INSTRUCTION_SIZE, size_t, rb_gc_mark, with_vm_lock}; use crate::cruby_methods; use crate::invariants::Invariants; -use crate::options::Options; use crate::asm::CodeBlock; +use crate::options::{get_option, rb_zjit_prepare_options}; +use crate::stats::{Counters, InsnCounters, SideExitLocations}; +use crate::virtualmem::CodePtr; +use std::collections::HashMap; +use std::ptr::null; +/// Shared trampoline to enter ZJIT. Not null when ZJIT is enabled. #[allow(non_upper_case_globals)] #[unsafe(no_mangle)] -pub static mut rb_zjit_enabled_p: bool = false; +pub static mut rb_zjit_entry: *const u8 = null(); /// Like rb_zjit_enabled_p, but for Rust code. pub fn zjit_enabled_p() -> bool { - unsafe { rb_zjit_enabled_p } + unsafe { rb_zjit_entry != null() } } /// Global state needed for code generation @@ -18,8 +26,14 @@ pub struct ZJITState { /// Inline code block (fast path) code_block: CodeBlock, - /// ZJIT command-line options - options: Options, + /// ZJIT statistics + counters: Counters, + + /// Side-exit counters + exit_counters: InsnCounters, + + /// Send fallback counters + send_fallback_counters: InsnCounters, /// Assumptions that require invalidation invariants: Invariants, @@ -29,72 +43,130 @@ pub struct ZJITState { /// Properties of core library methods method_annotations: cruby_methods::Annotations, + + /// Trampoline to side-exit without restoring PC or the stack + exit_trampoline: CodePtr, + + /// Trampoline to side-exit and increment exit_compilation_failure + exit_trampoline_with_counter: CodePtr, + + /// Trampoline to call function_stub_hit + function_stub_hit_trampoline: CodePtr, + + /// Counter pointers for full frame C functions + full_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for un-annotated C functions + not_annotated_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for all calls to any kind of C function from JIT code + ccall_counter_pointers: HashMap<String, Box<u64>>, + + /// Locations of side exists within generated code + exit_locations: Option<SideExitLocations>, +} + +/// Tracks the initialization progress +enum InitializationState { + Uninitialized, + + /// At boot time, rb_zjit_init will be called regardless of whether + /// ZJIT is enabled, in this phase we initialize any states that must + /// be captured at during boot. + Initialized(cruby_methods::Annotations), + + /// When ZJIT is enabled, either during boot with `--zjit`, or lazily + /// at a later time with `RubyVM::ZJIT.enable`, we perform the rest + /// of the initialization steps and produce the `ZJITState` instance. + Enabled(ZJITState), + + /// Indicates that ZJITState::init has panicked. Should never be + /// encountered in practice since we abort immediately when that + /// happens. + Panicked, } /// Private singleton instance of the codegen globals -static mut ZJIT_STATE: Option<ZJITState> = None; +static mut ZJIT_STATE: InitializationState = InitializationState::Uninitialized; impl ZJITState { - /// Initialize the ZJIT globals, given options allocated by rb_zjit_init_options() - pub fn init(options: Options) { - #[cfg(not(test))] - let cb = { - use crate::cruby::*; - - let exec_mem_size: usize = 64 * 1024 * 1024; // TODO: implement the option - let virt_block: *mut u8 = unsafe { rb_zjit_reserve_addr_space(64 * 1024 * 1024) }; - - // Memory protection syscalls need page-aligned addresses, so check it here. Assuming - // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the - // page size in bytes is a power of two 2¹⁹ or smaller. This is because the user - // requested size is half of mem_option × 2²⁰ as it's in MiB. - // - // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB - // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_zjit_get_page_size() }; - assert_eq!( - virt_block as usize % page_size as usize, 0, - "Start of virtual address block should be page-aligned", - ); + /// Initialize the ZJIT globals. Return the address of the JIT entry trampoline. + pub fn init() -> *const u8 { + use InitializationState::*; + let initialization_state = unsafe { + std::mem::replace(&mut ZJIT_STATE, Panicked) + }; + + let Initialized(method_annotations) = initialization_state else { + panic!("rb_zjit_init was never called"); + }; + + let mut cb = { + use crate::options::*; use crate::virtualmem::*; - use std::ptr::NonNull; use std::rc::Rc; use std::cell::RefCell; - let mem_block = VirtualMem::new( - crate::virtualmem::sys::SystemAllocator {}, - page_size, - NonNull::new(virt_block).unwrap(), - exec_mem_size, - 64 * 1024 * 1024, // TODO: support the option - ); + let mem_block = VirtualMem::alloc(get_option!(exec_mem_bytes), Some(get_option!(mem_bytes))); let mem_block = Rc::new(RefCell::new(mem_block)); - CodeBlock::new(mem_block.clone(), options.dump_disasm) + CodeBlock::new(mem_block.clone(), get_option!(dump_disasm)) + }; + + let entry_trampoline = gen_entry_trampoline(&mut cb).unwrap().raw_ptr(&cb); + let exit_trampoline = gen_exit_trampoline(&mut cb).unwrap(); + let function_stub_hit_trampoline = gen_function_stub_hit_trampoline(&mut cb).unwrap(); + + let exit_locations = if get_option!(trace_side_exits).is_some() { + Some(SideExitLocations::default()) + } else { + None }; - #[cfg(test)] - let cb = CodeBlock::new_dummy(); // Initialize the codegen globals instance let zjit_state = ZJITState { code_block: cb, - options, + counters: Counters::default(), + exit_counters: [0; VM_INSTRUCTION_SIZE as usize], + send_fallback_counters: [0; VM_INSTRUCTION_SIZE as usize], invariants: Invariants::default(), assert_compiles: false, - method_annotations: cruby_methods::init() + method_annotations, + exit_trampoline, + function_stub_hit_trampoline, + exit_trampoline_with_counter: exit_trampoline, + full_frame_cfunc_counter_pointers: HashMap::new(), + not_annotated_frame_cfunc_counter_pointers: HashMap::new(), + ccall_counter_pointers: HashMap::new(), + exit_locations, }; - unsafe { ZJIT_STATE = Some(zjit_state); } + unsafe { ZJIT_STATE = Enabled(zjit_state); } + + // With --zjit-stats, use a different trampoline on function stub exits + // to count exit_compilation_failure. Note that the trampoline code depends + // on the counter, so ZJIT_STATE needs to be initialized first. + if get_option!(stats) { + let cb = ZJITState::get_code_block(); + let code_ptr = gen_exit_trampoline_with_counter(cb, exit_trampoline).unwrap(); + ZJITState::get_instance().exit_trampoline_with_counter = code_ptr; + } + + entry_trampoline } /// Return true if zjit_state has been initialized pub fn has_instance() -> bool { - unsafe { ZJIT_STATE.as_mut().is_some() } + matches!(unsafe { &ZJIT_STATE }, InitializationState::Enabled(_)) } /// Get a mutable reference to the codegen globals instance fn get_instance() -> &'static mut ZJITState { - unsafe { ZJIT_STATE.as_mut().unwrap() } + if let InitializationState::Enabled(instance) = unsafe { &mut ZJIT_STATE } { + instance + } else { + panic!("ZJITState::get_instance called when ZJIT is not enabled") + } } /// Get a mutable reference to the inline code block @@ -102,11 +174,6 @@ impl ZJITState { &mut ZJITState::get_instance().code_block } - /// Get a mutable reference to the options - pub fn get_options() -> &'static mut Options { - &mut ZJITState::get_instance().options - } - /// Get a mutable reference to the invariants pub fn get_invariants() -> &'static mut Invariants { &mut ZJITState::get_instance().invariants @@ -126,36 +193,357 @@ impl ZJITState { let instance = ZJITState::get_instance(); instance.assert_compiles = true; } + + /// Get a mutable reference to counters for ZJIT stats + pub fn get_counters() -> &'static mut Counters { + &mut ZJITState::get_instance().counters + } + + /// Get a mutable reference to side-exit counters + pub fn get_exit_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().exit_counters + } + + /// Get a mutable reference to fallback counters + pub fn get_send_fallback_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().send_fallback_counters + } + + /// Get a mutable reference to full frame cfunc counter pointers + pub fn get_not_inlined_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().full_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to non-annotated cfunc counter pointers + pub fn get_not_annotated_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().not_annotated_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to ccall counter pointers + pub fn get_ccall_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().ccall_counter_pointers + } + + /// Was --zjit-save-compiled-iseqs specified? + pub fn should_log_compiled_iseqs() -> bool { + get_option!(log_compiled_iseqs).is_some() + } + + /// Log the name of a compiled ISEQ to the file specified in options.log_compiled_iseqs + pub fn log_compile(iseq_name: String) { + assert!(ZJITState::should_log_compiled_iseqs()); + let filename = get_option!(log_compiled_iseqs).as_ref().unwrap(); + use std::io::Write; + let mut file = match std::fs::OpenOptions::new().create(true).append(true).open(filename) { + Ok(f) => f, + Err(e) => { + eprintln!("ZJIT: Failed to create file '{}': {}", filename.display(), e); + return; + } + }; + if let Err(e) = writeln!(file, "{iseq_name}") { + eprintln!("ZJIT: Failed to write to file '{}': {}", filename.display(), e); + } + } + + /// Check if we are allowed to compile a given ISEQ based on --zjit-allowed-iseqs + pub fn can_compile_iseq(iseq: cruby::IseqPtr) -> bool { + if let Some(ref allowed_iseqs) = get_option!(allowed_iseqs) { + let name = cruby::iseq_get_location(iseq, 0); + allowed_iseqs.contains(&name) + } else { + true // If no restrictions, allow all ISEQs + } + } + + /// Return a code pointer to the side-exit trampoline + pub fn get_exit_trampoline() -> CodePtr { + ZJITState::get_instance().exit_trampoline + } + + /// Return a code pointer to the exit trampoline for function stubs + pub fn get_exit_trampoline_with_counter() -> CodePtr { + ZJITState::get_instance().exit_trampoline_with_counter + } + + /// Return a code pointer to the function stub hit trampoline + pub fn get_function_stub_hit_trampoline() -> CodePtr { + ZJITState::get_instance().function_stub_hit_trampoline + } + + /// Get a mutable reference to the ZJIT raw samples Vec + pub fn get_raw_samples() -> Option<&'static mut Vec<VALUE>> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| &mut el.raw_samples) + } + + /// Get a mutable reference to the ZJIT line samples Vec. + pub fn get_line_samples() -> Option<&'static mut Vec<i32>> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| &mut el.line_samples) + } + + /// Get number of skipped samples. + pub fn get_skipped_samples() -> Option<&'static mut usize> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| &mut el.skipped_samples) + } + + /// Get number of skipped samples. + pub fn set_skipped_samples(n: usize) -> Option<()> { + ZJITState::get_instance().exit_locations.as_mut().map(|el| el.skipped_samples = n) + } } -/// Initialize ZJIT, given options allocated by rb_zjit_init_options() +/// Initialize ZJIT at boot. This is called even if ZJIT is disabled. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_init(options: *const u8) { +pub extern "C" fn rb_zjit_init(zjit_enabled: bool) { + use InitializationState::*; + + debug_assert!( + matches!(unsafe { &ZJIT_STATE }, Uninitialized), + "rb_zjit_init should only be called once during boot", + ); + + // Initialize IDs and method annotations. + // cruby_methods::init() must be called at boot, + // as cmes could have been re-defined after boot. + cruby::ids::init(); + + let method_annotations = cruby_methods::init(); + + unsafe { ZJIT_STATE = Initialized(method_annotations); } + + // If --zjit, enable ZJIT immediately + if zjit_enabled { + zjit_enable(); + } +} + +/// Enable ZJIT compilation. +fn zjit_enable() { + // TODO: call RubyVM::ZJIT::call_jit_hooks here + // Catch panics to avoid UB for unwinding into C frames. // See https://doc.rust-lang.org/nomicon/exception-safety.html let result = std::panic::catch_unwind(|| { - cruby::ids::init(); - - let options = unsafe { Box::from_raw(options as *mut Options) }; - ZJITState::init(*options); - std::mem::drop(options); + // Initialize ZJIT states + let zjit_entry = ZJITState::init(); + // Install a panic hook for ZJIT rb_bug_panic_hook(); + // Discard the instruction count for boot which we never compile + unsafe { rb_vm_insn_count = 0; } + // ZJIT enabled and initialized successfully - assert!(unsafe{ !rb_zjit_enabled_p }); - unsafe { rb_zjit_enabled_p = true; } + assert!(unsafe{ rb_zjit_entry == null() }); + unsafe { rb_zjit_entry = zjit_entry; } }); if result.is_err() { - println!("ZJIT: zjit_init() panicked. Aborting."); + println!("ZJIT: zjit_enable() panicked. Aborting."); std::process::abort(); } } +/// Enable ZJIT compilation, returning Qtrue if ZJIT was previously disabled +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_enable(_ec: EcPtr, _self: VALUE) -> VALUE { + with_vm_lock(src_loc!(), || { + // Options would not have been initialized during boot if no flags were specified + rb_zjit_prepare_options(); + + // Initialize and enable ZJIT + zjit_enable(); + + // Add "+ZJIT" to RUBY_DESCRIPTION + unsafe { + unsafe extern "C" { + fn ruby_set_zjit_description(); + } + ruby_set_zjit_description(); + } + + Qtrue + }) +} + /// Assert that any future ZJIT compilation will return a function pointer (not fail to compile) #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_assert_compiles(_ec: EcPtr, _self: VALUE) -> VALUE { ZJITState::enable_assert_compiles(); Qnil } + +/// Call `rb_profile_frames` and write the result into buffers to be consumed by `rb_zjit_record_exit_stack`. +fn record_profiling_frames() -> (i32, Vec<VALUE>, Vec<i32>) { + // Stackprof uses a buffer of length 2048 when collating the frames into statistics. + // Since eventually the collected information will be used by Stackprof, collect only + // 2048 frames at a time. + // https://github.com/tmm1/stackprof/blob/5d832832e4afcb88521292d6dfad4a9af760ef7c/ext/stackprof/stackprof.c#L21 + const BUFF_LEN: usize = 2048; + + let mut frames_buffer = vec![VALUE(0_usize); BUFF_LEN]; + let mut lines_buffer = vec![0; BUFF_LEN]; + + let stack_length = unsafe { + rb_profile_frames( + 0, + BUFF_LEN as i32, + frames_buffer.as_mut_ptr(), + lines_buffer.as_mut_ptr(), + ) + }; + + // Trim at `stack_length` since anything past it is redundant + frames_buffer.truncate(stack_length as usize); + lines_buffer.truncate(stack_length as usize); + + (stack_length, frames_buffer, lines_buffer) +} + +/// Write samples in `frames_buffer` and `lines_buffer` from profiling into +/// `raw_samples` and `line_samples`. Also write opcode, number of frames, +/// and stack size to be consumed by Stackprof. +fn write_exit_stack_samples( + raw_samples: &'static mut Vec<VALUE>, + line_samples: &'static mut Vec<i32>, + frames_buffer: &[VALUE], + lines_buffer: &[i32], + stack_length: i32, + exit_pc: *const VALUE, +) { + raw_samples.push(VALUE(stack_length as usize)); + line_samples.push(stack_length); + + // Push frames and their lines in reverse order. + for i in (0..stack_length as usize).rev() { + raw_samples.push(frames_buffer[i]); + line_samples.push(lines_buffer[i]); + } + + // Get the opcode from instruction handler at exit PC. + let exit_opcode = unsafe { rb_vm_insn_addr2opcode((*exit_pc).as_ptr()) }; + raw_samples.push(VALUE(exit_opcode as usize)); + // Push a dummy line number since we don't know where this insn is from. + line_samples.push(0); + + // Push number of times seen onto the stack. + raw_samples.push(VALUE(1usize)); + line_samples.push(1); +} + +fn try_increment_existing_stack( + raw_samples: &mut [VALUE], + line_samples: &mut [i32], + frames_buffer: &[VALUE], + stack_length: i32, + samples_length: usize, +) -> bool { + let prev_stack_len_index = raw_samples.len() - samples_length; + let prev_stack_len = i64::from(raw_samples[prev_stack_len_index]); + + if prev_stack_len == stack_length as i64 { + // Check if all stack lengths match and all frames are identical + let frames_match = (0..stack_length).all(|i| { + let current_frame = frames_buffer[stack_length as usize - 1 - i as usize]; + let prev_frame = raw_samples[prev_stack_len_index + i as usize + 1]; + current_frame == prev_frame + }); + + if frames_match { + let counter_idx = raw_samples.len() - 1; + let new_count = i64::from(raw_samples[counter_idx]) + 1; + + raw_samples[counter_idx] = VALUE(new_count as usize); + line_samples[counter_idx] = new_count as i32; + return true; + } + } + false +} + +/// Record a backtrace with ZJIT side exits +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_record_exit_stack(exit_pc: *const VALUE) { + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return; + } + + // When `trace_side_exits_sample_interval` is zero, then the feature is disabled. + if get_option!(trace_side_exits_sample_interval) != 0 { + // If `trace_side_exits_sample_interval` is set, then can safely unwrap + // both `get_skipped_samples` and `set_skipped_samples`. + let skipped_samples = *ZJITState::get_skipped_samples().unwrap(); + if skipped_samples < get_option!(trace_side_exits_sample_interval) { + // Skip sample and increment counter. + ZJITState::set_skipped_samples(skipped_samples + 1).unwrap(); + return; + } else { + ZJITState::set_skipped_samples(0).unwrap(); + } + } + + let (stack_length, frames_buffer, lines_buffer) = record_profiling_frames(); + + // Can safely unwrap since `trace_side_exits` must be true at this point + let zjit_raw_samples = ZJITState::get_raw_samples().unwrap(); + let zjit_line_samples = ZJITState::get_line_samples().unwrap(); + assert_eq!(zjit_raw_samples.len(), zjit_line_samples.len()); + + // Represents pushing the stack length, the instruction opcode, and the sample count. + const SAMPLE_METADATA_SIZE: usize = 3; + let samples_length = (stack_length as usize) + SAMPLE_METADATA_SIZE; + + // If zjit_raw_samples is greater than or equal to the current length of the samples + // we might have seen this stack trace previously. + if zjit_raw_samples.len() >= samples_length + && try_increment_existing_stack( + zjit_raw_samples, + zjit_line_samples, + &frames_buffer, + stack_length, + samples_length, + ) + { + return; + } + + write_exit_stack_samples( + zjit_raw_samples, + zjit_line_samples, + &frames_buffer, + &lines_buffer, + stack_length, + exit_pc, + ); +} + +/// Mark `raw_samples` so they can be used by rb_zjit_add_frame. +pub fn gc_mark_raw_samples() { + // Return if ZJIT is not enabled + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return; + } + + let mut idx: size_t = 0; + let zjit_raw_samples = ZJITState::get_raw_samples().unwrap(); + + while idx < zjit_raw_samples.len() as size_t { + let num = zjit_raw_samples[idx as usize]; + let mut i = 0; + idx += 1; + + // Mark the zjit_raw_samples at the given index. These represent + // the data that needs to be GC'd which are the current frames. + while i < i32::from(num) { + unsafe { rb_gc_mark(zjit_raw_samples[idx as usize]); } + i += 1; + idx += 1; + } + + // Increase index for exit instruction. + idx += 1; + // Increase index for bookeeping value (number of times we've seen this + // row in a stack). + idx += 1; + } +} diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 06913522c4..506bd82686 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -1,9 +1,944 @@ -// Maxime would like to rebuild an improved stats system -// Individual stats should be tagged as always available, or only available in stats mode -// We could also tag which stats are fallback or exit counters, etc. Maybe even tag units? -// -// Comptime vs Runtime stats? - -pub fn zjit_alloc_size() -> usize { - 0 // TODO: report the actual memory usage +//! Counters and associated methods for events when ZJIT is run. + +use std::time::Instant; +use std::sync::atomic::Ordering; +use crate::options::OPTIONS; + +#[cfg(feature = "stats_allocator")] +#[path = "../../jit/src/lib.rs"] +mod jit; + +use crate::{cruby::*, hir::ParseError, options::get_option, state::{zjit_enabled_p, ZJITState}}; + +macro_rules! make_counters { + ( + default { + $($default_counter_name:ident,)+ + } + exit { + $($exit_counter_name:ident,)+ + } + dynamic_send { + $($dynamic_send_counter_name:ident,)+ + } + optimized_send { + $($optimized_send_counter_name:ident,)+ + } + dynamic_setivar { + $($dynamic_setivar_counter_name:ident,)+ + } + dynamic_getivar { + $($dynamic_getivar_counter_name:ident,)+ + } + dynamic_definedivar { + $($dynamic_definedivar_counter_name:ident,)+ + } + $($counter_name:ident,)+ + ) => { + /// Struct containing the counter values + #[derive(Default, Debug)] + pub struct Counters { + $(pub $default_counter_name: u64,)+ + $(pub $exit_counter_name: u64,)+ + $(pub $dynamic_send_counter_name: u64,)+ + $(pub $optimized_send_counter_name: u64,)+ + $(pub $dynamic_setivar_counter_name: u64,)+ + $(pub $dynamic_getivar_counter_name: u64,)+ + $(pub $dynamic_definedivar_counter_name: u64,)+ + $(pub $counter_name: u64,)+ + } + + /// Enum to represent a counter + #[allow(non_camel_case_types)] + #[derive(Clone, Copy, PartialEq, Eq, Debug)] + pub enum Counter { + $($default_counter_name,)+ + $($exit_counter_name,)+ + $($dynamic_send_counter_name,)+ + $($optimized_send_counter_name,)+ + $($dynamic_setivar_counter_name,)+ + $($dynamic_getivar_counter_name,)+ + $($dynamic_definedivar_counter_name,)+ + $($counter_name,)+ + } + + impl Counter { + pub fn name(&self) -> &'static str { + match self { + $( Counter::$default_counter_name => stringify!($default_counter_name), )+ + $( Counter::$exit_counter_name => stringify!($exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => stringify!($dynamic_send_counter_name), )+ + $( Counter::$optimized_send_counter_name => stringify!($optimized_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => stringify!($dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => stringify!($dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => stringify!($dynamic_definedivar_counter_name), )+ + $( Counter::$counter_name => stringify!($counter_name), )+ + } + } + + pub fn get(name: &str) -> Option<Counter> { + match name { + $( stringify!($default_counter_name) => Some(Counter::$default_counter_name), )+ + $( stringify!($exit_counter_name) => Some(Counter::$exit_counter_name), )+ + $( stringify!($dynamic_send_counter_name) => Some(Counter::$dynamic_send_counter_name), )+ + $( stringify!($optimized_send_counter_name) => Some(Counter::$optimized_send_counter_name), )+ + $( stringify!($dynamic_setivar_counter_name) => Some(Counter::$dynamic_setivar_counter_name), )+ + $( stringify!($dynamic_getivar_counter_name) => Some(Counter::$dynamic_getivar_counter_name), )+ + $( stringify!($dynamic_definedivar_counter_name) => Some(Counter::$dynamic_definedivar_counter_name), )+ + $( stringify!($counter_name) => Some(Counter::$counter_name), )+ + _ => None, + } + } + } + + /// Map a counter to a pointer + pub fn counter_ptr(counter: Counter) -> *mut u64 { + let counters = $crate::state::ZJITState::get_counters(); + match counter { + $( Counter::$default_counter_name => std::ptr::addr_of_mut!(counters.$default_counter_name), )+ + $( Counter::$exit_counter_name => std::ptr::addr_of_mut!(counters.$exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_definedivar_counter_name), )+ + $( Counter::$optimized_send_counter_name => std::ptr::addr_of_mut!(counters.$optimized_send_counter_name), )+ + $( Counter::$counter_name => std::ptr::addr_of_mut!(counters.$counter_name), )+ + } + } + + /// List of counters that are available without --zjit-stats. + /// They are incremented only by `incr_counter()` and don't use `gen_incr_counter()`. + pub const DEFAULT_COUNTERS: &'static [Counter] = &[ + $( Counter::$default_counter_name, )+ + ]; + + /// List of other counters that are summed as side_exit_count. + pub const EXIT_COUNTERS: &'static [Counter] = &[ + $( Counter::$exit_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_send_count. + pub const DYNAMIC_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_send_counter_name, )+ + ]; + + /// List of other counters that are summed as optimized_send_count. + pub const OPTIMIZED_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$optimized_send_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_setivar_count. + pub const DYNAMIC_SETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_setivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_getivar_count. + pub const DYNAMIC_GETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_getivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_definedivar_count. + pub const DYNAMIC_DEFINEDIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_definedivar_counter_name, )+ + ]; + + /// List of other counters that are available only for --zjit-stats. + pub const OTHER_COUNTERS: &'static [Counter] = &[ + $( Counter::$counter_name, )+ + ]; + } +} + +// Declare all the counters we track +make_counters! { + // Default counters that are available without --zjit-stats + default { + compiled_iseq_count, + failed_iseq_count, + + compile_time_ns, + profile_time_ns, + gc_time_ns, + invalidation_time_ns, + } + + // Exit counters that are summed as side_exit_count + exit { + // exit_: Side exits reasons + exit_compile_error, + exit_unhandled_newarray_send_min, + exit_unhandled_newarray_send_hash, + exit_unhandled_newarray_send_pack, + exit_unhandled_newarray_send_pack_buffer, + exit_unhandled_newarray_send_unknown, + exit_unhandled_duparray_send, + exit_unhandled_tailcall, + exit_unhandled_splat, + exit_unhandled_kwarg, + exit_unhandled_block_arg, + exit_unknown_special_variable, + exit_unhandled_hir_insn, + exit_unhandled_yarv_insn, + exit_fixnum_add_overflow, + exit_fixnum_sub_overflow, + exit_fixnum_mult_overflow, + exit_fixnum_lshift_overflow, + exit_fixnum_mod_by_zero, + exit_fixnum_div_by_zero, + exit_box_fixnum_overflow, + exit_guard_type_failure, + exit_guard_type_not_failure, + exit_guard_bit_equals_failure, + exit_guard_int_equals_failure, + exit_guard_shape_failure, + exit_expandarray_failure, + exit_guard_not_frozen_failure, + exit_guard_not_shared_failure, + exit_guard_less_failure, + exit_guard_greater_eq_failure, + exit_guard_super_method_entry, + exit_patchpoint_bop_redefined, + exit_patchpoint_method_redefined, + exit_patchpoint_stable_constant_names, + exit_patchpoint_no_tracepoint, + exit_patchpoint_no_ep_escape, + exit_patchpoint_single_ractor_mode, + exit_patchpoint_no_singleton_class, + exit_callee_side_exit, + exit_obj_to_string_fallback, + exit_interrupt, + exit_stackoverflow, + exit_block_param_proxy_modified, + exit_block_param_proxy_not_iseq_or_ifunc, + exit_too_many_keyword_parameters, + } + + // Send fallback counters that are summed as dynamic_send_count + dynamic_send { + // send_fallback_: Fallback reasons for send-ish instructions + send_fallback_send_without_block_polymorphic, + send_fallback_send_without_block_megamorphic, + send_fallback_send_without_block_no_profiles, + send_fallback_send_without_block_cfunc_not_variadic, + send_fallback_send_without_block_cfunc_array_variadic, + send_fallback_send_without_block_not_optimized_method_type, + send_fallback_send_without_block_not_optimized_method_type_optimized, + send_fallback_send_without_block_not_optimized_need_permission, + send_fallback_too_many_args_for_lir, + send_fallback_send_without_block_bop_redefined, + send_fallback_send_without_block_operands_not_fixnum, + send_fallback_send_without_block_direct_keyword_mismatch, + send_fallback_send_without_block_direct_optional_keywords, + send_fallback_send_without_block_direct_keyword_count_mismatch, + send_fallback_send_without_block_direct_missing_keyword, + send_fallback_send_polymorphic, + send_fallback_send_megamorphic, + send_fallback_send_no_profiles, + send_fallback_send_not_optimized_method_type, + send_fallback_send_not_optimized_need_permission, + send_fallback_ccall_with_frame_too_many_args, + send_fallback_argc_param_mismatch, + // The call has at least one feature on the caller or callee side + // that the optimizer does not support. + send_fallback_one_or_more_complex_arg_pass, + // Caller has keyword arguments but callee doesn't expect them. + send_fallback_unexpected_keyword_args, + // Singleton class previously created for receiver class. + send_fallback_singleton_class_seen, + send_fallback_bmethod_non_iseq_proc, + send_fallback_obj_to_string_not_string, + send_fallback_send_cfunc_variadic, + send_fallback_send_cfunc_array_variadic, + send_fallback_super_call_with_block, + send_fallback_super_class_not_found, + send_fallback_super_complex_args_pass, + send_fallback_super_fallback_no_profile, + send_fallback_super_not_optimized_method_type, + send_fallback_super_polymorphic, + send_fallback_super_target_not_found, + send_fallback_super_target_complex_args_pass, + send_fallback_cannot_send_direct, + send_fallback_uncategorized, + } + + // Optimized send counters that are summed as optimized_send_count + optimized_send { + iseq_optimized_send_count, + inline_cfunc_optimized_send_count, + inline_iseq_optimized_send_count, + non_variadic_cfunc_optimized_send_count, + variadic_cfunc_optimized_send_count, + } + + // Ivar fallback counters that are summed as dynamic_setivar_count + dynamic_setivar { + // setivar_fallback_: Fallback reasons for dynamic setivar instructions + setivar_fallback_not_monomorphic, + setivar_fallback_immediate, + setivar_fallback_not_t_object, + setivar_fallback_too_complex, + setivar_fallback_frozen, + setivar_fallback_shape_transition, + setivar_fallback_new_shape_too_complex, + setivar_fallback_new_shape_needs_extension, + } + + // Ivar fallback counters that are summed as dynamic_getivar_count + dynamic_getivar { + // getivar_fallback_: Fallback reasons for dynamic getivar instructions + getivar_fallback_not_monomorphic, + getivar_fallback_immediate, + getivar_fallback_not_t_object, + getivar_fallback_too_complex, + } + + // Ivar fallback counters that are summed as dynamic_definedivar_count + dynamic_definedivar { + // definedivar_fallback_: Fallback reasons for dynamic definedivar instructions + definedivar_fallback_not_monomorphic, + definedivar_fallback_immediate, + definedivar_fallback_not_t_object, + definedivar_fallback_too_complex, + } + + // compile_error_: Compile error reasons + compile_error_iseq_version_limit_reached, + compile_error_iseq_stack_too_large, + compile_error_exception_handler, + compile_error_out_of_memory, + compile_error_jit_to_jit_optional, + compile_error_register_spill_on_ccall, + compile_error_register_spill_on_alloc, + compile_error_parse_stack_underflow, + compile_error_parse_malformed_iseq, + compile_error_parse_not_allowed, + compile_error_validation_block_has_no_terminator, + compile_error_validation_terminator_not_at_end, + compile_error_validation_mismatched_block_arity, + compile_error_validation_jump_target_not_in_rpo, + compile_error_validation_operand_not_defined, + compile_error_validation_duplicate_instruction, + compile_error_validation_type_check_failure, + compile_error_validation_misc_validation_error, + + // unhandled_hir_insn_: Unhandled HIR instructions + unhandled_hir_insn_array_max, + unhandled_hir_insn_fixnum_div, + unhandled_hir_insn_throw, + unhandled_hir_insn_invokebuiltin, + unhandled_hir_insn_unknown, + + // The number of times YARV instructions are executed on JIT code + zjit_insn_count, + + // Method call def_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_iseq, + unspecialized_send_without_block_def_type_cfunc, + unspecialized_send_without_block_def_type_attrset, + unspecialized_send_without_block_def_type_ivar, + unspecialized_send_without_block_def_type_bmethod, + unspecialized_send_without_block_def_type_zsuper, + unspecialized_send_without_block_def_type_alias, + unspecialized_send_without_block_def_type_undef, + unspecialized_send_without_block_def_type_not_implemented, + unspecialized_send_without_block_def_type_optimized, + unspecialized_send_without_block_def_type_missing, + unspecialized_send_without_block_def_type_refined, + unspecialized_send_without_block_def_type_null, + + // Method call optimized_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_optimized_send, + unspecialized_send_without_block_def_type_optimized_call, + unspecialized_send_without_block_def_type_optimized_block_call, + unspecialized_send_without_block_def_type_optimized_struct_aref, + unspecialized_send_without_block_def_type_optimized_struct_aset, + + // Method call def_type related to send fallback to dynamic dispatch + unspecialized_send_def_type_iseq, + unspecialized_send_def_type_cfunc, + unspecialized_send_def_type_attrset, + unspecialized_send_def_type_ivar, + unspecialized_send_def_type_bmethod, + unspecialized_send_def_type_zsuper, + unspecialized_send_def_type_alias, + unspecialized_send_def_type_undef, + unspecialized_send_def_type_not_implemented, + unspecialized_send_def_type_optimized, + unspecialized_send_def_type_missing, + unspecialized_send_def_type_refined, + unspecialized_send_def_type_null, + + // Super call def_type related to send fallback to dynamic dispatch + unspecialized_super_def_type_iseq, + unspecialized_super_def_type_cfunc, + unspecialized_super_def_type_attrset, + unspecialized_super_def_type_ivar, + unspecialized_super_def_type_bmethod, + unspecialized_super_def_type_zsuper, + unspecialized_super_def_type_alias, + unspecialized_super_def_type_undef, + unspecialized_super_def_type_not_implemented, + unspecialized_super_def_type_optimized, + unspecialized_super_def_type_missing, + unspecialized_super_def_type_refined, + unspecialized_super_def_type_null, + + // Unsupported parameter features + complex_arg_pass_param_rest, + complex_arg_pass_param_post, + complex_arg_pass_param_kw_opt, + complex_arg_pass_param_kwrest, + complex_arg_pass_param_block, + complex_arg_pass_param_forwardable, + + // Unsupported caller side features + complex_arg_pass_caller_splat, + complex_arg_pass_caller_blockarg, + complex_arg_pass_caller_kwarg, + complex_arg_pass_caller_kw_splat, + complex_arg_pass_caller_tailcall, + complex_arg_pass_caller_super, + complex_arg_pass_caller_zsuper, + complex_arg_pass_caller_forwarding, + + // Writes to the VM frame + vm_write_pc_count, + vm_write_sp_count, + vm_write_locals_count, + vm_write_stack_count, + vm_write_to_parent_iseq_local_count, + vm_read_from_parent_iseq_local_count, + // TODO(max): Implement + // vm_reify_stack_count, + + // The number of times we ran a dynamic check + guard_type_count, + guard_shape_count, + + invokeblock_handler_monomorphic_iseq, + invokeblock_handler_monomorphic_ifunc, + invokeblock_handler_monomorphic_other, + invokeblock_handler_polymorphic, + invokeblock_handler_megamorphic, + invokeblock_handler_no_profiles, +} + +/// Increase a counter by a specified amount +pub fn incr_counter_by(counter: Counter, amount: u64) { + let ptr = counter_ptr(counter); + unsafe { *ptr += amount; } +} + +/// Decrease a counter by a specified amount +pub fn decr_counter_by(counter: Counter, amount: u64) { + let ptr = counter_ptr(counter); + unsafe { *ptr -= amount; } +} + +/// Increment a counter by its identifier +macro_rules! incr_counter { + ($counter_name:ident) => { + $crate::stats::incr_counter_by($crate::stats::Counter::$counter_name, 1) + } +} +pub(crate) use incr_counter; + +/// The number of side exits from each YARV instruction +pub type InsnCounters = [u64; VM_INSTRUCTION_SIZE as usize]; + +/// Return a raw pointer to the exit counter for a given YARV opcode +pub fn exit_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let exit_counters = ZJITState::get_exit_counters(); + unsafe { exit_counters.get_unchecked_mut(opcode as usize) } +} + +/// Return a raw pointer to the fallback counter for a given YARV opcode +pub fn send_fallback_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let fallback_counters = ZJITState::get_send_fallback_counters(); + unsafe { fallback_counters.get_unchecked_mut(opcode as usize) } +} + +/// Reason why ZJIT failed to produce any JIT code +#[derive(Clone, Debug, PartialEq)] +pub enum CompileError { + IseqVersionLimitReached, + IseqStackTooLarge, + ExceptionHandler, + OutOfMemory, + ParseError(ParseError), +} + +/// Return a raw pointer to the exit counter for a given CompileError +pub fn exit_counter_for_compile_error(compile_error: &CompileError) -> Counter { + use crate::hir::ParseError::*; + use crate::hir::ValidationError::*; + use crate::stats::CompileError::*; + use crate::stats::Counter::*; + match compile_error { + IseqVersionLimitReached => compile_error_iseq_version_limit_reached, + IseqStackTooLarge => compile_error_iseq_stack_too_large, + ExceptionHandler => compile_error_exception_handler, + OutOfMemory => compile_error_out_of_memory, + ParseError(parse_error) => match parse_error { + StackUnderflow(_) => compile_error_parse_stack_underflow, + MalformedIseq(_) => compile_error_parse_malformed_iseq, + NotAllowed => compile_error_parse_not_allowed, + Validation(validation) => match validation { + BlockHasNoTerminator(_) => compile_error_validation_block_has_no_terminator, + TerminatorNotAtEnd(_, _, _) => compile_error_validation_terminator_not_at_end, + MismatchedBlockArity(_, _, _) => compile_error_validation_mismatched_block_arity, + JumpTargetNotInRPO(_) => compile_error_validation_jump_target_not_in_rpo, + OperandNotDefined(_, _, _) => compile_error_validation_operand_not_defined, + DuplicateInstruction(_, _) => compile_error_validation_duplicate_instruction, + MismatchedOperandType(..) => compile_error_validation_type_check_failure, + MiscValidationError(..) => compile_error_validation_misc_validation_error, + }, + } + } +} + +pub fn exit_counter_for_unhandled_hir_insn(insn: &crate::hir::Insn) -> Counter { + use crate::hir::Insn::*; + use crate::stats::Counter::*; + match insn { + ArrayMax { .. } => unhandled_hir_insn_array_max, + FixnumDiv { .. } => unhandled_hir_insn_fixnum_div, + Throw { .. } => unhandled_hir_insn_throw, + InvokeBuiltin { .. } => unhandled_hir_insn_invokebuiltin, + _ => unhandled_hir_insn_unknown, + } +} + +pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { + use crate::hir::SideExitReason::*; + use crate::hir::CallType::*; + use crate::hir::Invariant; + use crate::stats::Counter::*; + match reason { + UnhandledNewarraySend(send_type) => match send_type { + VM_OPT_NEWARRAY_SEND_MIN => exit_unhandled_newarray_send_min, + VM_OPT_NEWARRAY_SEND_HASH => exit_unhandled_newarray_send_hash, + VM_OPT_NEWARRAY_SEND_PACK => exit_unhandled_newarray_send_pack, + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => exit_unhandled_newarray_send_pack_buffer, + _ => exit_unhandled_newarray_send_unknown, + } + UnhandledDuparraySend(_) => exit_unhandled_duparray_send, + UnhandledCallType(Tailcall) => exit_unhandled_tailcall, + UnhandledCallType(Splat) => exit_unhandled_splat, + UnhandledCallType(Kwarg) => exit_unhandled_kwarg, + UnknownSpecialVariable(_) => exit_unknown_special_variable, + UnhandledHIRInsn(_) => exit_unhandled_hir_insn, + UnhandledYARVInsn(_) => exit_unhandled_yarv_insn, + UnhandledBlockArg => exit_unhandled_block_arg, + FixnumAddOverflow => exit_fixnum_add_overflow, + FixnumSubOverflow => exit_fixnum_sub_overflow, + FixnumMultOverflow => exit_fixnum_mult_overflow, + FixnumLShiftOverflow => exit_fixnum_lshift_overflow, + FixnumModByZero => exit_fixnum_mod_by_zero, + FixnumDivByZero => exit_fixnum_div_by_zero, + BoxFixnumOverflow => exit_box_fixnum_overflow, + GuardType(_) => exit_guard_type_failure, + GuardTypeNot(_) => exit_guard_type_not_failure, + GuardShape(_) => exit_guard_shape_failure, + ExpandArray => exit_expandarray_failure, + GuardNotFrozen => exit_guard_not_frozen_failure, + GuardNotShared => exit_guard_not_shared_failure, + GuardLess => exit_guard_less_failure, + GuardGreaterEq => exit_guard_greater_eq_failure, + GuardSuperMethodEntry => exit_guard_super_method_entry, + CalleeSideExit => exit_callee_side_exit, + ObjToStringFallback => exit_obj_to_string_fallback, + Interrupt => exit_interrupt, + StackOverflow => exit_stackoverflow, + BlockParamProxyModified => exit_block_param_proxy_modified, + BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, + TooManyKeywordParameters => exit_too_many_keyword_parameters, + PatchPoint(Invariant::BOPRedefined { .. }) + => exit_patchpoint_bop_redefined, + PatchPoint(Invariant::MethodRedefined { .. }) + => exit_patchpoint_method_redefined, + PatchPoint(Invariant::StableConstantNames { .. }) + => exit_patchpoint_stable_constant_names, + PatchPoint(Invariant::NoTracePoint) + => exit_patchpoint_no_tracepoint, + PatchPoint(Invariant::NoEPEscape(_)) + => exit_patchpoint_no_ep_escape, + PatchPoint(Invariant::SingleRactorMode) + => exit_patchpoint_single_ractor_mode, + PatchPoint(Invariant::NoSingletonClass { .. }) + => exit_patchpoint_no_singleton_class, + } +} + +pub fn exit_counter_ptr(reason: crate::hir::SideExitReason) -> *mut u64 { + let counter = side_exit_counter(reason); + counter_ptr(counter) +} + +pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter { + use crate::hir::SendFallbackReason::*; + use crate::stats::Counter::*; + match reason { + SendWithoutBlockPolymorphic => send_fallback_send_without_block_polymorphic, + SendWithoutBlockMegamorphic => send_fallback_send_without_block_megamorphic, + SendWithoutBlockNoProfiles => send_fallback_send_without_block_no_profiles, + SendWithoutBlockCfuncNotVariadic => send_fallback_send_without_block_cfunc_not_variadic, + SendWithoutBlockCfuncArrayVariadic => send_fallback_send_without_block_cfunc_array_variadic, + SendWithoutBlockNotOptimizedMethodType(_) => send_fallback_send_without_block_not_optimized_method_type, + SendWithoutBlockNotOptimizedMethodTypeOptimized(_) + => send_fallback_send_without_block_not_optimized_method_type_optimized, + SendWithoutBlockNotOptimizedNeedPermission + => send_fallback_send_without_block_not_optimized_need_permission, + TooManyArgsForLir => send_fallback_too_many_args_for_lir, + SendWithoutBlockBopRedefined => send_fallback_send_without_block_bop_redefined, + SendWithoutBlockOperandsNotFixnum => send_fallback_send_without_block_operands_not_fixnum, + SendWithoutBlockDirectKeywordMismatch => send_fallback_send_without_block_direct_keyword_mismatch, + SendWithoutBlockDirectOptionalKeywords => send_fallback_send_without_block_direct_optional_keywords, + SendWithoutBlockDirectKeywordCountMismatch=> send_fallback_send_without_block_direct_keyword_count_mismatch, + SendWithoutBlockDirectMissingKeyword => send_fallback_send_without_block_direct_missing_keyword, + SendPolymorphic => send_fallback_send_polymorphic, + SendMegamorphic => send_fallback_send_megamorphic, + SendNoProfiles => send_fallback_send_no_profiles, + SendCfuncVariadic => send_fallback_send_cfunc_variadic, + SendCfuncArrayVariadic => send_fallback_send_cfunc_array_variadic, + ComplexArgPass => send_fallback_one_or_more_complex_arg_pass, + UnexpectedKeywordArgs => send_fallback_unexpected_keyword_args, + SingletonClassSeen => send_fallback_singleton_class_seen, + ArgcParamMismatch => send_fallback_argc_param_mismatch, + BmethodNonIseqProc => send_fallback_bmethod_non_iseq_proc, + SendNotOptimizedMethodType(_) => send_fallback_send_not_optimized_method_type, + SendNotOptimizedNeedPermission => send_fallback_send_not_optimized_need_permission, + CCallWithFrameTooManyArgs => send_fallback_ccall_with_frame_too_many_args, + ObjToStringNotString => send_fallback_obj_to_string_not_string, + SuperCallWithBlock => send_fallback_super_call_with_block, + SuperClassNotFound => send_fallback_super_class_not_found, + SuperComplexArgsPass => send_fallback_super_complex_args_pass, + SuperNoProfiles => send_fallback_super_fallback_no_profile, + SuperNotOptimizedMethodType(_) => send_fallback_super_not_optimized_method_type, + SuperPolymorphic => send_fallback_super_polymorphic, + SuperTargetNotFound => send_fallback_super_target_not_found, + SuperTargetComplexArgsPass => send_fallback_super_target_complex_args_pass, + Uncategorized(_) => send_fallback_uncategorized, + } +} + +pub fn send_without_block_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_without_block_def_type_iseq, + Cfunc => unspecialized_send_without_block_def_type_cfunc, + Attrset => unspecialized_send_without_block_def_type_attrset, + Ivar => unspecialized_send_without_block_def_type_ivar, + Bmethod => unspecialized_send_without_block_def_type_bmethod, + Zsuper => unspecialized_send_without_block_def_type_zsuper, + Alias => unspecialized_send_without_block_def_type_alias, + Undefined => unspecialized_send_without_block_def_type_undef, + NotImplemented => unspecialized_send_without_block_def_type_not_implemented, + Optimized => unspecialized_send_without_block_def_type_optimized, + Missing => unspecialized_send_without_block_def_type_missing, + Refined => unspecialized_send_without_block_def_type_refined, + Null => unspecialized_send_without_block_def_type_null, + } +} + +pub fn send_without_block_fallback_counter_for_optimized_method_type(method_type: crate::hir::OptimizedMethodType) -> Counter { + use crate::hir::OptimizedMethodType::*; + use crate::stats::Counter::*; + + match method_type { + Send => unspecialized_send_without_block_def_type_optimized_send, + Call => unspecialized_send_without_block_def_type_optimized_call, + BlockCall => unspecialized_send_without_block_def_type_optimized_block_call, + StructAref => unspecialized_send_without_block_def_type_optimized_struct_aref, + StructAset => unspecialized_send_without_block_def_type_optimized_struct_aset, + } +} + +pub fn send_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_def_type_iseq, + Cfunc => unspecialized_send_def_type_cfunc, + Attrset => unspecialized_send_def_type_attrset, + Ivar => unspecialized_send_def_type_ivar, + Bmethod => unspecialized_send_def_type_bmethod, + Zsuper => unspecialized_send_def_type_zsuper, + Alias => unspecialized_send_def_type_alias, + Undefined => unspecialized_send_def_type_undef, + NotImplemented => unspecialized_send_def_type_not_implemented, + Optimized => unspecialized_send_def_type_optimized, + Missing => unspecialized_send_def_type_missing, + Refined => unspecialized_send_def_type_refined, + Null => unspecialized_send_def_type_null, + } +} + +pub fn send_fallback_counter_for_super_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_super_def_type_iseq, + Cfunc => unspecialized_super_def_type_cfunc, + Attrset => unspecialized_super_def_type_attrset, + Ivar => unspecialized_super_def_type_ivar, + Bmethod => unspecialized_super_def_type_bmethod, + Zsuper => unspecialized_super_def_type_zsuper, + Alias => unspecialized_super_def_type_alias, + Undefined => unspecialized_super_def_type_undef, + NotImplemented => unspecialized_super_def_type_not_implemented, + Optimized => unspecialized_super_def_type_optimized, + Missing => unspecialized_super_def_type_missing, + Refined => unspecialized_super_def_type_refined, + Null => unspecialized_super_def_type_null, + } +} + +/// Primitive called in zjit.rb. Zero out all the counters. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_reset_stats_bang(_ec: EcPtr, _self: VALUE) -> VALUE { + let counters = ZJITState::get_counters(); + let exit_counters = ZJITState::get_exit_counters(); + + // Reset all counters to zero + *counters = Counters::default(); + + // Reset exit counters for YARV instructions + exit_counters.as_mut_slice().fill(0); + + Qnil +} + +/// Return a Hash object that contains ZJIT statistics +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> VALUE { + if !zjit_enabled_p() { + return Qnil; + } + + macro_rules! set_stat { + ($hash:ident, $key:expr, $value:expr) => { + let key = rust_str_to_sym($key); + if key == target_key { + return $value; + } else if $hash != Qnil { + #[allow(unused_unsafe)] + unsafe { rb_hash_aset($hash, key, $value); } + } + }; + } + + macro_rules! set_stat_usize { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, VALUE::fixnum_from_usize($value as usize)) + } + } + + macro_rules! set_stat_f64 { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, unsafe { rb_float_new($value) }) + } + } + + let hash = if target_key.nil_p() { + unsafe { rb_hash_new() } + } else { + Qnil + }; + + // Set default counters + for &counter in DEFAULT_COUNTERS { + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + } + + // Memory usage stats + let code_region_bytes = ZJITState::get_code_block().mapped_region_size(); + set_stat_usize!(hash, "code_region_bytes", code_region_bytes); + set_stat_usize!(hash, "zjit_alloc_bytes", zjit_alloc_bytes()); + set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + zjit_alloc_bytes()); + + // End of default stats. Every counter beyond this is provided only for --zjit-stats. + if !get_option!(stats) { + return hash; + } + + // Set other stats-only counters + for &counter in OTHER_COUNTERS { + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + } + + // Set side-exit counters for each SideExitReason + let mut side_exit_count = 0; + for &counter in EXIT_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + side_exit_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "side_exit_count", side_exit_count); + + // Set side-exit counters for UnhandledYARVInsn + let exit_counters = ZJITState::get_exit_counters(); + for (op_idx, count) in exit_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "unhandled_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Set send fallback counters for each DynamicSendReason + let mut dynamic_send_count = 0; + for &counter in DYNAMIC_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_send_count", dynamic_send_count); + + // Set optimized send counters + let mut optimized_send_count = 0; + for &counter in OPTIMIZED_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + optimized_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "optimized_send_count", optimized_send_count); + set_stat_usize!(hash, "send_count", dynamic_send_count + optimized_send_count); + + // Set send fallback counters for each setivar fallback reason + let mut dynamic_setivar_count = 0; + for &counter in DYNAMIC_SETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_setivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_setivar_count", dynamic_setivar_count); + + // Set send fallback counters for each getivar fallback reason + let mut dynamic_getivar_count = 0; + for &counter in DYNAMIC_GETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_getivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_getivar_count", dynamic_getivar_count); + + // Set send fallback counters for each definedivar fallback reason + let mut dynamic_definedivar_count = 0; + for &counter in DYNAMIC_DEFINEDIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_definedivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_definedivar_count", dynamic_definedivar_count); + + // Set send fallback counters for Uncategorized + let send_fallback_counters = ZJITState::get_send_fallback_counters(); + for (op_idx, count) in send_fallback_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "uncategorized_fallback_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Only ZJIT_STATS builds support rb_vm_insn_count + if unsafe { rb_vm_insn_count } > 0 { + let vm_insn_count = unsafe { rb_vm_insn_count }; + set_stat_usize!(hash, "vm_insn_count", vm_insn_count); + + let zjit_insn_count = ZJITState::get_counters().zjit_insn_count; + let total_insn_count = vm_insn_count + zjit_insn_count; + set_stat_usize!(hash, "total_insn_count", total_insn_count); + + set_stat_f64!(hash, "ratio_in_zjit", 100.0 * zjit_insn_count as f64 / total_insn_count as f64); + } + + // Set not inlined cfunc counters + let not_inlined_cfuncs = ZJITState::get_not_inlined_cfunc_counter_pointers(); + for (signature, counter) in not_inlined_cfuncs.iter() { + let key_string = format!("not_inlined_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set not annotated cfunc counters + let not_annotated_cfuncs = ZJITState::get_not_annotated_cfunc_counter_pointers(); + for (signature, counter) in not_annotated_cfuncs.iter() { + let key_string = format!("not_annotated_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set ccall counters + let ccall = ZJITState::get_ccall_counter_pointers(); + for (signature, counter) in ccall.iter() { + let key_string = format!("ccall_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + hash +} + +/// Measure the time taken by func() and add that to zjit_compile_time. +pub fn with_time_stat<F, R>(counter: Counter, func: F) -> R where F: FnOnce() -> R { + let start = Instant::now(); + let ret = func(); + let nanos = Instant::now().duration_since(start).as_nanos(); + incr_counter_by(counter, nanos as u64); + ret +} + +/// The number of bytes ZJIT has allocated on the Rust heap. +pub fn zjit_alloc_bytes() -> usize { + jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) +} + +/// Struct of arrays for --zjit-trace-exits. +#[derive(Default)] +pub struct SideExitLocations { + /// Control frames of method entries. + pub raw_samples: Vec<VALUE>, + /// Line numbers of the iseq caller. + pub line_samples: Vec<i32>, + /// Skipped samples + pub skipped_samples: usize +} + +/// Primitive called in zjit.rb +/// +/// Check if trace_exits generation is enabled. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_trace_exit_locations_enabled_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.trace_side_exits.is_some()) { + Qtrue + } else { + Qfalse + } +} + +/// Call the C function to parse the raw_samples and line_samples +/// into raw, lines, and frames hash for RubyVM::YJIT.exit_locations. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_get_exit_locations(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return Qnil; + } + + // Can safely unwrap since `trace_side_exits` must be true at this point + let zjit_raw_samples = ZJITState::get_raw_samples().unwrap(); + let zjit_line_samples = ZJITState::get_line_samples().unwrap(); + + assert_eq!(zjit_raw_samples.len(), zjit_line_samples.len()); + + // zjit_raw_samples and zjit_line_samples are the same length so + // pass only one of the lengths in the C function. + let samples_len = zjit_raw_samples.len() as i32; + + unsafe { + rb_zjit_exit_locations_dict( + zjit_raw_samples.as_mut_ptr(), + zjit_line_samples.as_mut_ptr(), + samples_len + ) + } } diff --git a/zjit/src/ttycolors.rs b/zjit/src/ttycolors.rs new file mode 100644 index 0000000000..f325772431 --- /dev/null +++ b/zjit/src/ttycolors.rs @@ -0,0 +1,31 @@ +use std::io::IsTerminal; + +pub fn stdout_supports_colors() -> bool { + std::io::stdout().is_terminal() +} + +#[cfg_attr(not(feature = "disasm"), allow(dead_code))] +#[derive(Copy, Clone, Debug)] +pub struct TerminalColor { + pub bold_begin: &'static str, + pub bold_end: &'static str, +} + +pub static TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "\x1b[1m", + bold_end: "\x1b[22m", +}; + +pub static NON_TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "", + bold_end: "", +}; + +/// Terminal escape codes for colors, font weight, etc. Only enabled if stdout is a TTY. +pub fn get_colors() -> &'static TerminalColor { + if stdout_supports_colors() { + &TTY_TERMINAL_COLOR + } else { + &NON_TTY_TERMINAL_COLOR + } +} diff --git a/zjit/src/virtualmem.rs b/zjit/src/virtualmem.rs index d76c3d76d3..9741a7b138 100644 --- a/zjit/src/virtualmem.rs +++ b/zjit/src/virtualmem.rs @@ -4,15 +4,11 @@ // benefit. use std::ptr::NonNull; +use crate::cruby::*; +use crate::stats::zjit_alloc_bytes; -use crate::stats::zjit_alloc_size; - -#[cfg(not(test))] pub type VirtualMem = VirtualMemory<sys::SystemAllocator>; -#[cfg(test)] -pub type VirtualMem = VirtualMemory<tests::TestingAllocator>; - /// Memory for generated executable machine code. When not testing, we reserve address space for /// the entire region upfront and map physical memory into the reserved address space as needed. On /// Linux, this is basically done using an `mmap` with `PROT_NONE` upfront and gradually using @@ -30,7 +26,7 @@ pub struct VirtualMemory<A: Allocator> { region_size_bytes: usize, /// mapped_region_bytes + zjit_alloc_size may not increase beyond this limit. - memory_limit_bytes: usize, + memory_limit_bytes: Option<usize>, /// Number of bytes per "page", memory protection permission can only be controlled at this /// granularity. @@ -62,7 +58,7 @@ pub trait Allocator { /// Pointer into a [VirtualMemory] represented as an offset from the base. /// Note: there is no NULL constant for [CodePtr]. You should use `Option<CodePtr>` instead. -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Debug)] #[repr(C, packed)] pub struct CodePtr(u32); @@ -74,6 +70,13 @@ impl CodePtr { CodePtr(raw + bytes) } + /// Subtract bytes from the CodePtr + pub fn sub_bytes(self, bytes: usize) -> Self { + let CodePtr(raw) = self; + let bytes: u32 = bytes.try_into().unwrap(); + CodePtr(raw.saturating_sub(bytes)) + } + /// Note that the raw pointer might be dangling if there hasn't /// been any writes to it through the [VirtualMemory] yet. pub fn raw_ptr(self, base: &impl CodePtrBase) -> *const u8 { @@ -83,7 +86,7 @@ impl CodePtr { /// Get the address of the code pointer. pub fn raw_addr(self, base: &impl CodePtrBase) -> usize { - self.raw_ptr(base) as usize + self.raw_ptr(base).addr() } /// Get the offset component for the code pointer. Useful finding the distance between two @@ -103,6 +106,28 @@ pub enum WriteError { use WriteError::*; +impl VirtualMem { + /// Allocate a VirtualMem insntace with a requested size + pub fn alloc(exec_mem_bytes: usize, mem_bytes: Option<usize>) -> Self { + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_bytes as u32) }; + + // Memory protection syscalls need page-aligned addresses, so check it here. Assuming + // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the + // page size in bytes is a power of two 2¹⁹ or smaller. This is because the user + // requested size is half of mem_option × 2²⁰ as it's in MiB. + // + // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB + // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. + let page_size = unsafe { rb_jit_get_page_size() }; + assert_eq!( + virt_block as usize % page_size as usize, 0, + "Start of virtual address block should be page-aligned", + ); + + Self::new(sys::SystemAllocator {}, page_size, NonNull::new(virt_block).unwrap(), exec_mem_bytes, mem_bytes) + } +} + impl<A: Allocator> VirtualMemory<A> { /// Bring a part of the address space under management. pub fn new( @@ -110,7 +135,7 @@ impl<A: Allocator> VirtualMemory<A> { page_size: u32, virt_region_start: NonNull<u8>, region_size_bytes: usize, - memory_limit_bytes: usize, + memory_limit_bytes: Option<usize>, ) -> Self { assert_ne!(0, page_size); let page_size_bytes = page_size as usize; @@ -171,6 +196,12 @@ impl<A: Allocator> VirtualMemory<A> { let whole_region_end = start.wrapping_add(self.region_size_bytes); let alloc = &mut self.allocator; + // Ignore zjit_alloc_size() if self.memory_limit_bytes is None for testing + let mut required_region_bytes = page_addr + page_size - start as usize; + if self.memory_limit_bytes.is_some() { + required_region_bytes += zjit_alloc_bytes(); + } + assert!((start..=whole_region_end).contains(&mapped_region_end)); if (start..mapped_region_end).contains(&raw) { @@ -183,7 +214,7 @@ impl<A: Allocator> VirtualMemory<A> { self.current_write_page = Some(page_addr); } else if (start..whole_region_end).contains(&raw) && - (page_addr + page_size - start as usize) + zjit_alloc_size() < self.memory_limit_bytes { + required_region_bytes < self.memory_limit_bytes.unwrap_or(self.region_size_bytes) { // Writing to a brand new page let mapped_region_end_addr = mapped_region_end as usize; let alloc_size = page_addr - mapped_region_end_addr + page_size; @@ -227,6 +258,13 @@ impl<A: Allocator> VirtualMemory<A> { Ok(()) } + /// Return true if write_byte() can allocate a new page + pub fn can_allocate(&self) -> bool { + let memory_usage_bytes = self.mapped_region_bytes + zjit_alloc_bytes(); + let memory_limit_bytes = self.memory_limit_bytes.unwrap_or(self.region_size_bytes); + memory_usage_bytes + self.page_size_bytes < memory_limit_bytes + } + /// Make all the code in the region executable. Call this at the end of a write session. /// See [Self] for usual usage flow. pub fn mark_all_executable(&mut self) { @@ -269,7 +307,6 @@ impl<A: Allocator> CodePtrBase for VirtualMemory<A> { } /// Requires linking with CRuby to work -#[cfg(not(test))] pub mod sys { use crate::cruby::*; @@ -280,15 +317,15 @@ pub mod sys { impl super::Allocator for SystemAllocator { fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_writable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } } fn mark_executable(&mut self, ptr: *const u8, size: u32) { - unsafe { rb_zjit_mark_executable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } } fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_unused(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } } } } @@ -362,6 +399,12 @@ pub mod tests { // Fictional architecture where each page is 4 bytes long const PAGE_SIZE: usize = 4; fn new_dummy_virt_mem() -> VirtualMemory<TestingAllocator> { + unsafe { + if crate::options::OPTIONS.is_none() { + crate::options::OPTIONS = Some(crate::options::Options::default()); + } + } + let mem_size = PAGE_SIZE * 10; let alloc = TestingAllocator::new(mem_size); let mem_start: *const u8 = alloc.mem_start(); @@ -371,7 +414,7 @@ pub mod tests { PAGE_SIZE.try_into().unwrap(), NonNull::new(mem_start as *mut u8).unwrap(), mem_size, - 128 * 1024 * 1024, + None, ) } diff --git a/zjit/zjit.mk b/zjit/zjit.mk index 9107c15809..58b45d8787 100644 --- a/zjit/zjit.mk +++ b/zjit/zjit.mk @@ -1,9 +1,7 @@ # -*- mode: makefile-gmake; indent-tabs-mode: t -*- -# Show Cargo progress when doing `make V=1` -CARGO_VERBOSE_0 = -q -CARGO_VERBOSE_1 = -CARGO_VERBOSE = $(CARGO_VERBOSE_$(V)) +# Put no definitions when ZJIT isn't configured +ifneq ($(ZJIT_SUPPORT),no) ZJIT_SRC_FILES = $(wildcard \ $(top_srcdir)/zjit/Cargo.* \ @@ -11,29 +9,30 @@ ZJIT_SRC_FILES = $(wildcard \ $(top_srcdir)/zjit/src/*/*.rs \ $(top_srcdir)/zjit/src/*/*/*.rs \ $(top_srcdir)/zjit/src/*/*/*/*.rs \ + $(top_srcdir)/jit/src/lib.rs \ ) -# Because of Cargo cache, if the actual binary is not changed from the -# previous build, the mtime is preserved as the cached file. -# This means the target is not updated actually, and it will need to -# rebuild at the next build. -ZJIT_LIB_TOUCH = touch $@ +$(RUST_LIB): $(ZJIT_SRC_FILES) # Absolute path to match RUST_LIB rules to avoid picking # the "target" dir in the source directory through VPATH. BUILD_ZJIT_LIBS = $(TOP_BUILD_DIR)/$(ZJIT_LIBS) -# ZJIT_SUPPORT=yes when `configure` gets `--enable-zjit` -ifeq ($(ZJIT_SUPPORT),yes) +# In a ZJIT-only build (no YJIT) +ifneq ($(strip $(ZJIT_LIBS)),) $(BUILD_ZJIT_LIBS): $(ZJIT_SRC_FILES) $(ECHO) 'building Rust ZJIT (release mode)' - +$(Q) $(RUSTC) $(ZJIT_RUSTC_ARGS) - $(ZJIT_LIB_TOUCH) -endif + $(gnumake_recursive)$(Q) $(RUSTC) $(ZJIT_RUSTC_ARGS) +else ifneq ($(strip $(RLIB_DIR)),) # combo build +# Absolute path to avoid VPATH ambiguity +ZJIT_RLIB = $(TOP_BUILD_DIR)/$(RLIB_DIR)/libzjit.rlib -ifneq ($(ZJIT_SUPPORT),no) -$(RUST_LIB): $(ZJIT_SRC_FILES) -endif +$(ZJIT_RLIB): $(ZJIT_SRC_FILES) + $(ECHO) 'building $(@F)' + $(gnumake_recursive)$(Q) $(RUSTC) '-L$(@D)' --extern=jit $(ZJIT_RUSTC_ARGS) + +$(RUST_LIB): $(ZJIT_RLIB) +endif # ifneq ($(strip $(ZJIT_LIBS)),) # By using ZJIT_BENCH_OPTS instead of RUN_OPTS, you can skip passing the options to `make install` ZJIT_BENCH_OPTS = $(RUN_OPTS) --enable-gems @@ -49,8 +48,8 @@ update-zjit-bench: https://github.com/Shopify/zjit-bench zjit-bench $(GIT_OPTS) # Gives quick feedback about ZJIT. Not a replacement for a full test run. -.PHONY: zjit-test-all -zjit-test-all: +.PHONY: zjit-check +zjit-check: $(MAKE) zjit-test $(MAKE) test-all TESTS='$(top_srcdir)/test/ruby/test_zjit.rb' @@ -59,11 +58,18 @@ ZJIT_BINDGEN_DIFF_OPTS = # Generate Rust bindings. See source for details. # Needs `./configure --enable-zjit=dev` and Clang. ifneq ($(strip $(CARGO)),) # if configure found Cargo -.PHONY: zjit-bindgen zjit-bindgen-show-unused zjit-test zjit-test-lldb +.PHONY: zjit-bindgen zjit-bindgen-show-unused zjit-test zjit-test-update +.PHONY: zjit-test-debug zjit-test-lldb zjit-test-gdb zjit-test-rr zjit-bindgen: zjit.$(OBJEXT) ZJIT_SRC_ROOT_PATH='$(top_srcdir)' BINDGEN_JIT_NAME=zjit $(CARGO) run --manifest-path '$(top_srcdir)/zjit/bindgen/Cargo.toml' -- $(CFLAGS) $(XCFLAGS) $(CPPFLAGS) $(Q) if [ 'x$(HAVE_GIT)' = xyes ]; then $(GIT) -C "$(top_srcdir)" diff $(ZJIT_BINDGEN_DIFF_OPTS) zjit/src/cruby_bindings.inc.rs; fi +# Build env should roughly match what's used for miniruby to help with caching. +ZJIT_NEXTEST_ENV := RUBY_BUILD_DIR='$(TOP_BUILD_DIR)' \ + RUBY_LD_FLAGS='$(LDFLAGS) $(XLDFLAGS) $(MAINLIBS)' \ + MACOSX_DEPLOYMENT_TARGET=11.0 \ + CARGO_TARGET_DIR='$(CARGO_TARGET_DIR)' + # We need `cargo nextest` for its one-process-per execution execution model # since we can only boot the VM once per process. Normal `cargo test` # runs tests in threads and can't handle this. @@ -71,25 +77,52 @@ zjit-bindgen: zjit.$(OBJEXT) # On darwin, it's available through `brew install cargo-nextest`. See # https://nexte.st/docs/installation/pre-built-binaries/ otherwise. zjit-test: libminiruby.a - RUBY_BUILD_DIR='$(TOP_BUILD_DIR)' \ - RUBY_LD_FLAGS='$(LDFLAGS) $(XLDFLAGS) $(MAINLIBS)' \ - CARGO_TARGET_DIR='$(CARGO_TARGET_DIR)' \ - $(CARGO) nextest run --manifest-path '$(top_srcdir)/zjit/Cargo.toml' $(ZJIT_TESTS) - -# Run a ZJIT test written with Rust #[test] under LLDB -zjit-test-lldb: libminiruby.a + @set +e; \ + $(ZJIT_NEXTEST_ENV) $(CARGO) nextest run \ + --manifest-path '$(top_srcdir)/zjit/Cargo.toml' \ + --no-fail-fast \ + '--features=$(ZJIT_TEST_FEATURES)' \ + $(ZJIT_TESTS); \ + exit_code=$$?; \ + if [ -f '$(top_srcdir)/zjit/src/.hir.rs.pending-snap' ]; then \ + echo ""; \ + echo "Pending snapshots found. Accept with: make zjit-test-update"; \ + fi; \ + exit $$exit_code + +# Accept all pending snapshots (requires cargo-insta) +# Install with: cargo install cargo-insta +zjit-test-update: + @$(CARGO) insta --version >/dev/null 2>&1 || { echo "Error: cargo-insta is not installed. Install with: cargo install cargo-insta"; exit 1; } + @$(CARGO) insta accept --manifest-path '$(top_srcdir)/zjit/Cargo.toml' + +ZJIT_DEBUGGER = +ZJIT_DEBUGGER_OPTS = + +# Run a ZJIT test written with Rust #[test] under $(ZJIT_DEBUGGER) +zjit-test-debug: libminiruby.a $(Q)set -eu; \ if [ -z '$(ZJIT_TESTS)' ]; then \ echo "Please pass a ZJIT_TESTS=... filter to make."; \ echo "Many tests only work when it's the only test in the process."; \ exit 1; \ fi; \ - exe_path=`RUBY_BUILD_DIR='$(TOP_BUILD_DIR)' \ - RUBY_LD_FLAGS='$(LDFLAGS) $(XLDFLAGS) $(MAINLIBS)' \ - CARGO_TARGET_DIR='$(CARGO_TARGET_DIR)' \ + exe_path=`$(ZJIT_NEXTEST_ENV) \ $(CARGO) nextest list --manifest-path '$(top_srcdir)/zjit/Cargo.toml' --message-format json --list-type=binaries-only | \ $(BASERUBY) -rjson -e 'puts JSON.load(STDIN.read).dig("rust-binaries", "zjit", "binary-path")'`; \ - exec lldb $$exe_path -- --test-threads=1 $(ZJIT_TESTS) + exec $(ZJIT_DEBUGGER) $$exe_path $(ZJIT_DEBUGGER_OPTS) --test-threads=1 $(ZJIT_TESTS) + +# Run a ZJIT test written with Rust #[test] under LLDB +zjit-test-lldb: + $(Q) $(MAKE) zjit-test-debug ZJIT_DEBUGGER=lldb ZJIT_DEBUGGER_OPTS=-- + +# Run a ZJIT test written with Rust #[test] under GDB +zjit-test-gdb: libminiruby.a + $(Q) $(MAKE) zjit-test-debug ZJIT_DEBUGGER="gdb --args" + +# Run a ZJIT test written with Rust #[test] under rr-debugger +zjit-test-rr: libminiruby.a + $(Q) $(MAKE) zjit-test-debug ZJIT_DEBUGGER="rr record" # A library for booting miniruby in tests. # Why not use libruby-static.a for this? @@ -102,4 +135,6 @@ libminiruby.a: miniruby$(EXEEXT) $(Q) $(AR) $(ARFLAGS) $@ $(MINIOBJS) $(COMMONOBJS) libminiruby: libminiruby.a -endif + +endif # ifneq ($(strip $(CARGO)), +endif # ifneq ($(ZJIT_SUPPORT),no) |