diff options
Diffstat (limited to 'zjit/src')
54 files changed, 55361 insertions, 9538 deletions
diff --git a/zjit/src/asm/arm64/arg/sf.rs b/zjit/src/asm/arm64/arg/sf.rs index c2fd33302c..b6091821e9 100644 --- a/zjit/src/asm/arm64/arg/sf.rs +++ b/zjit/src/asm/arm64/arg/sf.rs @@ -13,7 +13,7 @@ impl From<u8> for Sf { match num_bits { 64 => Sf::Sf64, 32 => Sf::Sf32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/atomic.rs b/zjit/src/asm/arm64/inst/atomic.rs index dce9affedf..0917a4fd1c 100644 --- a/zjit/src/asm/arm64/inst/atomic.rs +++ b/zjit/src/asm/arm64/inst/atomic.rs @@ -14,7 +14,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/branch.rs b/zjit/src/asm/arm64/inst/branch.rs index 14fcb2e9fd..2db52e5d31 100644 --- a/zjit/src/asm/arm64/inst/branch.rs +++ b/zjit/src/asm/arm64/inst/branch.rs @@ -89,7 +89,7 @@ mod tests { #[test] fn test_ret() { let result: u32 = Branch::ret(30).into(); - assert_eq!(0xd65f03C0, result); + assert_eq!(0xd65f03c0, result); } #[test] diff --git a/zjit/src/asm/arm64/inst/load_literal.rs b/zjit/src/asm/arm64/inst/load_literal.rs index 817e893553..37b5f3c7a7 100644 --- a/zjit/src/asm/arm64/inst/load_literal.rs +++ b/zjit/src/asm/arm64/inst/load_literal.rs @@ -1,3 +1,5 @@ +#![allow(clippy::identity_op)] + use super::super::arg::{InstructionOffset, truncate_imm}; /// The size of the operands being operated on. @@ -13,7 +15,7 @@ impl From<u8> for Opc { match num_bits { 64 => Opc::Size64, 32 => Opc::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/load_register.rs b/zjit/src/asm/arm64/inst/load_register.rs index 3d94e8da1f..80813ffc87 100644 --- a/zjit/src/asm/arm64/inst/load_register.rs +++ b/zjit/src/asm/arm64/inst/load_register.rs @@ -25,7 +25,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/load_store.rs b/zjit/src/asm/arm64/inst/load_store.rs index e27909ae35..d38e851ed7 100644 --- a/zjit/src/asm/arm64/inst/load_store.rs +++ b/zjit/src/asm/arm64/inst/load_store.rs @@ -15,7 +15,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } @@ -124,6 +124,12 @@ impl LoadStore { pub fn sturh(rt: u8, rn: u8, imm9: i16) -> Self { Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size16 } } + + /// STURB (store register, byte, unscaled) + /// <https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STURH--Store-Register-Halfword--unscaled--?lang=en> + pub fn sturb(rt: u8, rn: u8, imm9: i16) -> Self { + Self { rt, rn, idx: Index::None, imm9, opc: Opc::STR, size: Size::Size8 } + } } /// <https://developer.arm.com/documentation/ddi0602/2022-03/Index-by-Encoding/Loads-and-Stores?lang=en> diff --git a/zjit/src/asm/arm64/inst/load_store_exclusive.rs b/zjit/src/asm/arm64/inst/load_store_exclusive.rs index 1106b4cb37..30cb663bdb 100644 --- a/zjit/src/asm/arm64/inst/load_store_exclusive.rs +++ b/zjit/src/asm/arm64/inst/load_store_exclusive.rs @@ -17,7 +17,7 @@ impl From<u8> for Size { match num_bits { 64 => Size::Size64, 32 => Size::Size32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/mod.rs b/zjit/src/asm/arm64/inst/mod.rs index bfffd914ef..270c784f27 100644 --- a/zjit/src/asm/arm64/inst/mod.rs +++ b/zjit/src/asm/arm64/inst/mod.rs @@ -26,6 +26,7 @@ mod sbfm; mod shift_imm; mod sys_reg; mod test_bit; +mod udf; pub use atomic::Atomic; pub use branch::Branch; @@ -52,3 +53,4 @@ pub use sbfm::SBFM; pub use shift_imm::ShiftImm; pub use sys_reg::SysReg; pub use test_bit::TestBit; +pub use udf::Udf; diff --git a/zjit/src/asm/arm64/inst/mov.rs b/zjit/src/asm/arm64/inst/mov.rs index 58877ae940..e9f9091713 100644 --- a/zjit/src/asm/arm64/inst/mov.rs +++ b/zjit/src/asm/arm64/inst/mov.rs @@ -27,7 +27,7 @@ impl From<u8> for Hw { 16 => Hw::LSL16, 32 => Hw::LSL32, 48 => Hw::LSL48, - _ => panic!("Invalid value for shift: {}", shift) + _ => panic!("Invalid value for shift: {shift}"), } } } @@ -145,14 +145,14 @@ mod tests { fn test_movk_shifted_16() { let inst = Mov::movk(0, 123, 16, 64); let result: u32 = inst.into(); - assert_eq!(0xf2A00f60, result); + assert_eq!(0xf2a00f60, result); } #[test] fn test_movk_shifted_32() { let inst = Mov::movk(0, 123, 32, 64); let result: u32 = inst.into(); - assert_eq!(0xf2C00f60, result); + assert_eq!(0xf2c00f60, result); } #[test] diff --git a/zjit/src/asm/arm64/inst/reg_pair.rs b/zjit/src/asm/arm64/inst/reg_pair.rs index 9bffcd8479..39a44c2416 100644 --- a/zjit/src/asm/arm64/inst/reg_pair.rs +++ b/zjit/src/asm/arm64/inst/reg_pair.rs @@ -26,7 +26,7 @@ impl From<u8> for Opc { match num_bits { 64 => Opc::Opc64, 32 => Opc::Opc32, - _ => panic!("Invalid number of bits: {}", num_bits) + _ => panic!("Invalid number of bits: {num_bits}"), } } } diff --git a/zjit/src/asm/arm64/inst/test_bit.rs b/zjit/src/asm/arm64/inst/test_bit.rs index f7aeca70fd..45f0c2317e 100644 --- a/zjit/src/asm/arm64/inst/test_bit.rs +++ b/zjit/src/asm/arm64/inst/test_bit.rs @@ -17,7 +17,7 @@ impl From<u8> for B5 { match bit_num { 0..=31 => B5::B532, 32..=63 => B5::B564, - _ => panic!("Invalid bit number: {}", bit_num) + _ => panic!("Invalid bit number: {bit_num}"), } } } diff --git a/zjit/src/asm/arm64/inst/udf.rs b/zjit/src/asm/arm64/inst/udf.rs new file mode 100644 index 0000000000..297d17ed62 --- /dev/null +++ b/zjit/src/asm/arm64/inst/udf.rs @@ -0,0 +1,52 @@ +/// The struct that represents an A64 permanently undefined instruction. +/// +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 09 08 | 07 06 05 04 | 03 02 01 00 | +/// | 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 | +/// | imm16..................................................| +/// +-------------+-------------+-------------+-------------+-------------+-------------+-------------+-------------+ +/// +pub struct Udf { + /// The immediate value encoded in the instruction + imm16: u16 +} + +impl Udf { + /// UDF - Permanently Undefined + /// <https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UDF--Permanently-Undefined-> + pub fn udf(imm16: u16) -> Self { + Self { imm16 } + } +} + +impl From<Udf> for u32 { + /// Convert an instruction into a 32-bit value. + fn from(inst: Udf) -> Self { + inst.imm16 as u32 + } +} + +impl From<Udf> for [u8; 4] { + /// Convert an instruction into a 4 byte array. + fn from(inst: Udf) -> [u8; 4] { + let result: u32 = inst.into(); + result.to_le_bytes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_udf() { + let result: u32 = Udf::udf(0).into(); + assert_eq!(0x00000000, result); + } + + #[test] + fn test_udf_imm() { + let result: u32 = Udf::udf(1).into(); + assert_eq!(0x00000001, result); + } +} diff --git a/zjit/src/asm/arm64/mod.rs b/zjit/src/asm/arm64/mod.rs index 0576b23090..b53f1cf673 100644 --- a/zjit/src/asm/arm64/mod.rs +++ b/zjit/src/asm/arm64/mod.rs @@ -1,4 +1,8 @@ #![allow(dead_code)] // For instructions and operands we're not currently using. +#![allow(clippy::upper_case_acronyms)] +#![allow(clippy::identity_op)] +#![allow(clippy::self_named_constructors)] +#![allow(clippy::unusual_byte_groupings)] use crate::asm::CodeBlock; @@ -14,7 +18,7 @@ pub use arg::*; pub use opnd::*; /// The extend type for register operands in extended register instructions. -/// It's the reuslt size is determined by the the destination register and +/// It's the result size is determined by the destination register and /// the source size interpreted using the last letter. #[derive(Clone, Copy)] pub enum ExtendType { @@ -237,7 +241,7 @@ pub fn asr(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { SBFM::asr(rd.reg_no, rn.reg_no, shift.try_into().unwrap(), rd.num_bits).into() }, - _ => panic!("Invalid operand combination to asr instruction: asr {:?}, {:?}, {:?}", rd, rn, shift), + _ => panic!("Invalid operand combination to asr instruction: asr {rd:?}, {rn:?}, {shift:?}"), }; cb.write_bytes(&bytes); @@ -317,6 +321,12 @@ pub fn brk(cb: &mut CodeBlock, imm16: A64Opnd) { cb.write_bytes(&bytes); } +/// UDF - permanently undefined instruction +pub fn udf(cb: &mut CodeBlock, imm16: u16) { + let bytes: [u8; 4] = Udf::udf(imm16).into(); + cb.write_bytes(&bytes); +} + /// CMP - compare rn and rm, update flags pub fn cmp(cb: &mut CodeBlock, rn: A64Opnd, rm: A64Opnd) { let bytes: [u8; 4] = match (rn, rm) { @@ -645,7 +655,7 @@ pub fn lsl(cb: &mut CodeBlock, rd: A64Opnd, rn: A64Opnd, shift: A64Opnd) { ShiftImm::lsl(rd.reg_no, rn.reg_no, uimm as u8, rd.num_bits).into() }, - _ => panic!("Invalid operands combination to lsl instruction") + _ => panic!("Invalid operands combination {rd:?} {rn:?} {shift:?} to lsl instruction") }; cb.write_bytes(&bytes); @@ -1022,7 +1032,21 @@ pub fn sturh(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { LoadStore::sturh(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() }, - _ => panic!("Invalid operand combination to stur instruction.") + _ => panic!("Invalid operand combination to sturh instruction: {rt:?}, {rn:?}") + }; + + cb.write_bytes(&bytes); +} + +pub fn sturb(cb: &mut CodeBlock, rt: A64Opnd, rn: A64Opnd) { + let bytes: [u8; 4] = match (rt, rn) { + (A64Opnd::Reg(rt), A64Opnd::Mem(rn)) => { + assert!(rn.num_bits == 8); + assert!(mem_disp_fits_bits(rn.disp), "Expected displacement {} to be 9 bits or less", rn.disp); + + LoadStore::sturb(rt.reg_no, rn.base_reg_no, rn.disp as i16).into() + }, + _ => panic!("Invalid operand combination to sturb instruction: {rt:?}, {rn:?}") }; cb.write_bytes(&bytes); @@ -1208,13 +1232,13 @@ fn cbz_cbnz(num_bits: u8, op: bool, offset: InstructionOffset, rt: u8) -> [u8; 4 #[cfg(test)] mod tests { use super::*; - use crate::assertions::assert_disasm; + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; - /// Check that the bytes for an instruction sequence match a hex string - fn check_bytes<R>(bytes: &str, run: R) where R: FnOnce(&mut super::CodeBlock) { + fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { let mut cb = super::CodeBlock::new_dummy(); run(&mut cb); - assert_eq!(format!("{:x}", cb), bytes); + cb } #[test] @@ -1242,94 +1266,130 @@ mod tests { #[test] fn test_add_reg() { - check_bytes("2000028b", |cb| add(cb, X0, X1, X2)); + let cb = compile(|cb| add(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028b"); } #[test] fn test_add_uimm() { - check_bytes("201c0091", |cb| add(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); } #[test] fn test_add_imm_positive() { - check_bytes("201c0091", |cb| add(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); } #[test] fn test_add_imm_negative() { - check_bytes("201c00d1", |cb| add(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| add(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); } #[test] fn test_adds_reg() { - check_bytes("200002ab", |cb| adds(cb, X0, X1, X2)); + let cb = compile(|cb| adds(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ab"); } #[test] fn test_adds_uimm() { - check_bytes("201c00b1", |cb| adds(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); } #[test] fn test_adds_imm_positive() { - check_bytes("201c00b1", |cb| adds(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); } #[test] fn test_adds_imm_negative() { - check_bytes("201c00f1", |cb| adds(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| adds(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); } #[test] fn test_adr() { - check_bytes("aa000010", |cb| adr(cb, X10, A64Opnd::new_imm(20))); + let cb = compile(|cb| adr(cb, X10, A64Opnd::new_imm(20))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adr x10, #0x14"); + assert_snapshot!(cb.hexdump(), @"aa000010"); } #[test] fn test_adrp() { - check_bytes("4a000090", |cb| adrp(cb, X10, A64Opnd::new_imm(0x8000))); + let cb = compile(|cb| adrp(cb, X10, A64Opnd::new_imm(0x8000))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adrp x10, #0x8000"); + assert_snapshot!(cb.hexdump(), @"4a000090"); } #[test] fn test_and_register() { - check_bytes("2000028a", |cb| and(cb, X0, X1, X2)); + let cb = compile(|cb| and(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"2000028a"); } #[test] fn test_and_immediate() { - check_bytes("20084092", |cb| and(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| and(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"20084092"); } #[test] fn test_and_32b_immediate() { - check_bytes("404c0012", |cb| and(cb, W0, W2, A64Opnd::new_uimm(0xfffff))); + let cb = compile(|cb| and(cb, W0, W2, A64Opnd::new_uimm(0xfffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and w0, w2, #0xfffff"); + assert_snapshot!(cb.hexdump(), @"404c0012"); } #[test] fn test_ands_register() { - check_bytes("200002ea", |cb| ands(cb, X0, X1, X2)); + let cb = compile(|cb| ands(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002ea"); } #[test] fn test_ands_immediate() { - check_bytes("200840f2", |cb| ands(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| ands(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ands x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"200840f2"); } #[test] fn test_asr() { - check_bytes("b4fe4a93", |cb| asr(cb, X20, X21, A64Opnd::new_uimm(10))); + let cb = compile(|cb| asr(cb, X20, X21, A64Opnd::new_uimm(10))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: asr x20, x21, #0xa"); + assert_snapshot!(cb.hexdump(), @"b4fe4a93"); } #[test] fn test_bcond() { let offset = InstructionOffset::from_insns(0x100); - check_bytes("01200054", |cb| bcond(cb, Condition::NE, offset)); + let cb = compile(|cb| bcond(cb, Condition::NE, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b.ne #0x400"); + assert_snapshot!(cb.hexdump(), @"01200054"); } #[test] fn test_b() { let offset = InstructionOffset::from_insns((1 << 25) - 1); - check_bytes("ffffff15", |cb| b(cb, offset)); + let cb = compile(|cb| b(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: b #0x7fffffc"); + assert_snapshot!(cb.hexdump(), @"ffffff15"); } #[test] @@ -1337,7 +1397,7 @@ mod tests { fn test_b_too_big() { // There are 26 bits available let offset = InstructionOffset::from_insns(1 << 25); - check_bytes("", |cb| b(cb, offset)); + compile(|cb| b(cb, offset)); } #[test] @@ -1345,13 +1405,15 @@ mod tests { fn test_b_too_small() { // There are 26 bits available let offset = InstructionOffset::from_insns(-(1 << 25) - 1); - check_bytes("", |cb| b(cb, offset)); + compile(|cb| b(cb, offset)); } #[test] fn test_bl() { let offset = InstructionOffset::from_insns(-(1 << 25)); - check_bytes("00000096", |cb| bl(cb, offset)); + let cb = compile(|cb| bl(cb, offset)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: bl #0xfffffffff8000000"); + assert_snapshot!(cb.hexdump(), @"00000096"); } #[test] @@ -1359,7 +1421,7 @@ mod tests { fn test_bl_too_big() { // There are 26 bits available let offset = InstructionOffset::from_insns(1 << 25); - check_bytes("", |cb| bl(cb, offset)); + compile(|cb| bl(cb, offset)); } #[test] @@ -1367,385 +1429,544 @@ mod tests { fn test_bl_too_small() { // There are 26 bits available let offset = InstructionOffset::from_insns(-(1 << 25) - 1); - check_bytes("", |cb| bl(cb, offset)); + compile(|cb| bl(cb, offset)); } #[test] fn test_blr() { - check_bytes("80023fd6", |cb| blr(cb, X20)); + let cb = compile(|cb| blr(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: blr x20"); + assert_snapshot!(cb.hexdump(), @"80023fd6"); } #[test] fn test_br() { - check_bytes("80021fd6", |cb| br(cb, X20)); + let cb = compile(|cb| br(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: br x20"); + assert_snapshot!(cb.hexdump(), @"80021fd6"); } #[test] fn test_cbz() { let offset = InstructionOffset::from_insns(-1); - check_bytes("e0ffffb4e0ffff34", |cb| { + let cb = compile(|cb| { cbz(cb, X0, offset); cbz(cb, W0, offset); }); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: cbz x0, #0xfffffffffffffffc + 0x4: cbz w0, #0 + "); + assert_snapshot!(cb.hexdump(), @"e0ffffb4e0ffff34"); } #[test] fn test_cbnz() { let offset = InstructionOffset::from_insns(2); - check_bytes("540000b554000035", |cb| { + let cb = compile(|cb| { cbnz(cb, X20, offset); cbnz(cb, W20, offset); }); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: cbnz x20, #8 + 0x4: cbnz w20, #0xc + "); + assert_snapshot!(cb.hexdump(), @"540000b554000035"); } #[test] fn test_brk_none() { - check_bytes("00003ed4", |cb| brk(cb, A64Opnd::None)); + let cb = compile(|cb| brk(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xf000"); + assert_snapshot!(cb.hexdump(), @"00003ed4"); } #[test] fn test_brk_uimm() { - check_bytes("c00120d4", |cb| brk(cb, A64Opnd::new_uimm(14))); + let cb = compile(|cb| brk(cb, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: brk #0xe"); + assert_snapshot!(cb.hexdump(), @"c00120d4"); } #[test] fn test_cmp_register() { - check_bytes("5f010beb", |cb| cmp(cb, X10, X11)); + let cb = compile(|cb| cmp(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, x11"); + assert_snapshot!(cb.hexdump(), @"5f010beb"); } #[test] fn test_cmp_immediate() { - check_bytes("5f3900f1", |cb| cmp(cb, X10, A64Opnd::new_uimm(14))); + let cb = compile(|cb| cmp(cb, X10, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp x10, #0xe"); + assert_snapshot!(cb.hexdump(), @"5f3900f1"); } #[test] fn test_csel() { - check_bytes("6a018c9a", |cb| csel(cb, X10, X11, X12, Condition::EQ)); + let cb = compile(|cb| csel(cb, X10, X11, X12, Condition::EQ)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: csel x10, x11, x12, eq"); + assert_snapshot!(cb.hexdump(), @"6a018c9a"); } #[test] fn test_eor_register() { - check_bytes("6a010cca", |cb| eor(cb, X10, X11, X12)); + let cb = compile(|cb| eor(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010cca"); } #[test] fn test_eor_immediate() { - check_bytes("6a0940d2", |cb| eor(cb, X10, X11, A64Opnd::new_uimm(7))); + let cb = compile(|cb| eor(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940d2"); } #[test] fn test_eor_32b_immediate() { - check_bytes("29040152", |cb| eor(cb, W9, W1, A64Opnd::new_uimm(0x80000001))); + let cb = compile(|cb| eor(cb, W9, W1, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor w9, w1, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"29040152"); } #[test] fn test_ldaddal() { - check_bytes("8b01eaf8", |cb| ldaddal(cb, X10, X11, X12)); + let cb = compile(|cb| ldaddal(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaddal x10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8b01eaf8"); } #[test] fn test_ldaxr() { - check_bytes("6afd5fc8", |cb| ldaxr(cb, X10, X11)); + let cb = compile(|cb| ldaxr(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldaxr x10, [x11]"); + assert_snapshot!(cb.hexdump(), @"6afd5fc8"); } #[test] fn test_ldp() { - check_bytes("8a2d4da9", |cb| ldp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| ldp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d4da9"); } #[test] fn test_ldp_pre() { - check_bytes("8a2dcda9", |cb| ldp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| ldp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2dcda9"); } #[test] fn test_ldp_post() { - check_bytes("8a2dcda8", |cb| ldp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| ldp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2dcda8"); } #[test] fn test_ldr() { - check_bytes("6a696cf8", |cb| ldr(cb, X10, X11, X12)); + let cb = compile(|cb| ldr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, x12]"); + assert_snapshot!(cb.hexdump(), @"6a696cf8"); } #[test] fn test_ldr_literal() { - check_bytes("40010058", |cb| ldr_literal(cb, X0, 10.into())); + let cb = compile(|cb| ldr_literal(cb, X0, 10.into())); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x0, #0x28"); + assert_snapshot!(cb.hexdump(), @"40010058"); } #[test] fn test_ldr_post() { - check_bytes("6a0541f8", |cb| ldr_post(cb, X10, A64Opnd::new_mem(64, X11, 16))); + let cb = compile(|cb| ldr_post(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11], #0x10"); + assert_snapshot!(cb.hexdump(), @"6a0541f8"); } #[test] fn test_ldr_pre() { - check_bytes("6a0d41f8", |cb| ldr_pre(cb, X10, A64Opnd::new_mem(64, X11, 16))); + let cb = compile(|cb| ldr_pre(cb, X10, A64Opnd::new_mem(64, X11, 16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x10, [x11, #0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d41f8"); } #[test] fn test_ldrh() { - check_bytes("6a194079", |cb| ldrh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| ldrh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a194079"); } #[test] fn test_ldrh_pre() { - check_bytes("6acd4078", |cb| ldrh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| ldrh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd4078"); } #[test] fn test_ldrh_post() { - check_bytes("6ac54078", |cb| ldrh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| ldrh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldrh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac54078"); } #[test] fn test_ldurh_memory() { - check_bytes("2a004078", |cb| ldurh(cb, W10, A64Opnd::new_mem(64, X1, 0))); - check_bytes("2ab04778", |cb| ldurh(cb, W10, A64Opnd::new_mem(64, X1, 123))); + let cb = compile(|cb| { + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 0)); + ldurh(cb, W10, A64Opnd::new_mem(64, X1, 123)); + }); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldurh w10, [x1] + 0x4: ldurh w10, [x1, #0x7b] + "); + assert_snapshot!(cb.hexdump(), @"2a0040782ab04778"); } #[test] fn test_ldur_memory() { - check_bytes("20b047f8", |cb| ldur(cb, X0, A64Opnd::new_mem(64, X1, 123))); + let cb = compile(|cb| ldur(cb, X0, A64Opnd::new_mem(64, X1, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"20b047f8"); } #[test] fn test_ldur_register() { - check_bytes("200040f8", |cb| ldur(cb, X0, X1)); + let cb = compile(|cb| ldur(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x1]"); + assert_snapshot!(cb.hexdump(), @"200040f8"); } #[test] fn test_ldursw() { - check_bytes("6ab187b8", |cb| ldursw(cb, X10, A64Opnd::new_mem(64, X11, 123))); + let cb = compile(|cb| ldursw(cb, X10, A64Opnd::new_mem(64, X11, 123))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldursw x10, [x11, #0x7b]"); + assert_snapshot!(cb.hexdump(), @"6ab187b8"); } #[test] fn test_lsl() { - check_bytes("6ac572d3", |cb| lsl(cb, X10, X11, A64Opnd::new_uimm(14))); + let cb = compile(|cb| lsl(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsl x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6ac572d3"); } #[test] fn test_lsr() { - check_bytes("6afd4ed3", |cb| lsr(cb, X10, X11, A64Opnd::new_uimm(14))); + let cb = compile(|cb| lsr(cb, X10, X11, A64Opnd::new_uimm(14))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lsr x10, x11, #0xe"); + assert_snapshot!(cb.hexdump(), @"6afd4ed3"); } #[test] fn test_mov_registers() { - check_bytes("ea030baa", |cb| mov(cb, X10, X11)); + let cb = compile(|cb| mov(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea030baa"); } #[test] fn test_mov_immediate() { - check_bytes("eaf300b2", |cb| mov(cb, X10, A64Opnd::new_uimm(0x5555555555555555))); + let cb = compile(|cb| mov(cb, X10, A64Opnd::new_uimm(0x5555555555555555))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x10, #0x5555555555555555"); + assert_snapshot!(cb.hexdump(), @"eaf300b2"); } #[test] fn test_mov_32b_immediate() { - check_bytes("ea070132", |cb| mov(cb, W10, A64Opnd::new_uimm(0x80000001))); + let cb = compile(|cb| mov(cb, W10, A64Opnd::new_uimm(0x80000001))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov w10, #-0x7fffffff"); + assert_snapshot!(cb.hexdump(), @"ea070132"); } #[test] fn test_mov_into_sp() { - check_bytes("1f000091", |cb| mov(cb, X31, X0)); + let cb = compile(|cb| mov(cb, X31, X0)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov sp, x0"); + assert_snapshot!(cb.hexdump(), @"1f000091"); } #[test] fn test_mov_from_sp() { - check_bytes("e0030091", |cb| mov(cb, X0, X31)); + let cb = compile(|cb| mov(cb, X0, X31)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, sp"); + assert_snapshot!(cb.hexdump(), @"e0030091"); } #[test] fn test_movk() { - check_bytes("600fa0f2", |cb| movk(cb, X0, A64Opnd::new_uimm(123), 16)); + let cb = compile(|cb| movk(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: movk x0, #0x7b, lsl #16"); + assert_snapshot!(cb.hexdump(), @"600fa0f2"); } #[test] fn test_movn() { - check_bytes("600fa092", |cb| movn(cb, X0, A64Opnd::new_uimm(123), 16)); + let cb = compile(|cb| movn(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #-0x7b0001"); + assert_snapshot!(cb.hexdump(), @"600fa092"); } #[test] fn test_movz() { - check_bytes("600fa0d2", |cb| movz(cb, X0, A64Opnd::new_uimm(123), 16)); + let cb = compile(|cb| movz(cb, X0, A64Opnd::new_uimm(123), 16)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #0x7b0000"); + assert_snapshot!(cb.hexdump(), @"600fa0d2"); } #[test] fn test_mrs() { - check_bytes("0a423bd5", |cb| mrs(cb, X10, SystemRegister::NZCV)); + let cb = compile(|cb| mrs(cb, X10, SystemRegister::NZCV)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mrs x10, nzcv"); + assert_snapshot!(cb.hexdump(), @"0a423bd5"); } #[test] fn test_msr() { - check_bytes("0a421bd5", |cb| msr(cb, SystemRegister::NZCV, X10)); + let cb = compile(|cb| msr(cb, SystemRegister::NZCV, X10)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: msr nzcv, x10"); + assert_snapshot!(cb.hexdump(), @"0a421bd5"); } #[test] fn test_mul() { - check_bytes("6a7d0c9b", |cb| mul(cb, X10, X11, X12)); + let cb = compile(|cb| mul(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mul x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a7d0c9b"); } #[test] fn test_mvn() { - check_bytes("ea032baa", |cb| mvn(cb, X10, X11)); + let cb = compile(|cb| mvn(cb, X10, X11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mvn x10, x11"); + assert_snapshot!(cb.hexdump(), @"ea032baa"); } #[test] fn test_nop() { - check_bytes("1f2003d5", |cb| nop(cb)); + let cb = compile(nop); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: nop"); + assert_snapshot!(cb.hexdump(), @"1f2003d5"); } #[test] fn test_orn() { - check_bytes("6a012caa", |cb| orn(cb, X10, X11, X12)); + let cb = compile(|cb| orn(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orn x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a012caa"); } #[test] fn test_orr_register() { - check_bytes("6a010caa", |cb| orr(cb, X10, X11, X12)); + let cb = compile(|cb| orr(cb, X10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, x12"); + assert_snapshot!(cb.hexdump(), @"6a010caa"); } #[test] fn test_orr_immediate() { - check_bytes("6a0940b2", |cb| orr(cb, X10, X11, A64Opnd::new_uimm(7))); + let cb = compile(|cb| orr(cb, X10, X11, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr x10, x11, #7"); + assert_snapshot!(cb.hexdump(), @"6a0940b2"); } #[test] fn test_orr_32b_immediate() { - check_bytes("6a010032", |cb| orr(cb, W10, W11, A64Opnd::new_uimm(1))); + let cb = compile(|cb| orr(cb, W10, W11, A64Opnd::new_uimm(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: orr w10, w11, #1"); + assert_snapshot!(cb.hexdump(), @"6a010032"); } #[test] fn test_ret_none() { - check_bytes("c0035fd6", |cb| ret(cb, A64Opnd::None)); + let cb = compile(|cb| ret(cb, A64Opnd::None)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c0035fd6"); } #[test] fn test_ret_register() { - check_bytes("80025fd6", |cb| ret(cb, X20)); + let cb = compile(|cb| ret(cb, X20)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret x20"); + assert_snapshot!(cb.hexdump(), @"80025fd6"); } #[test] fn test_stlxr() { - check_bytes("8bfd0ac8", |cb| stlxr(cb, W10, X11, X12)); + let cb = compile(|cb| stlxr(cb, W10, X11, X12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stlxr w10, x11, [x12]"); + assert_snapshot!(cb.hexdump(), @"8bfd0ac8"); } #[test] fn test_stp() { - check_bytes("8a2d0da9", |cb| stp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| stp(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]"); + assert_snapshot!(cb.hexdump(), @"8a2d0da9"); } #[test] fn test_stp_pre() { - check_bytes("8a2d8da9", |cb| stp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| stp_pre(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12, #0xd0]!"); + assert_snapshot!(cb.hexdump(), @"8a2d8da9"); } #[test] fn test_stp_post() { - check_bytes("8a2d8da8", |cb| stp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + let cb = compile(|cb| stp_post(cb, X10, X11, A64Opnd::new_mem(64, X12, 208))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stp x10, x11, [x12], #0xd0"); + assert_snapshot!(cb.hexdump(), @"8a2d8da8"); } #[test] fn test_str_post() { - check_bytes("6a051ff8", |cb| str_post(cb, X10, A64Opnd::new_mem(64, X11, -16))); + let cb = compile(|cb| str_post(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11], #0xfffffffffffffff0"); + assert_snapshot!(cb.hexdump(), @"6a051ff8"); } #[test] fn test_str_pre() { - check_bytes("6a0d1ff8", |cb| str_pre(cb, X10, A64Opnd::new_mem(64, X11, -16))); + let cb = compile(|cb| str_pre(cb, X10, A64Opnd::new_mem(64, X11, -16))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: str x10, [x11, #-0x10]!"); + assert_snapshot!(cb.hexdump(), @"6a0d1ff8"); } #[test] fn test_strh() { - check_bytes("6a190079", |cb| strh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| strh(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]"); + assert_snapshot!(cb.hexdump(), @"6a190079"); } #[test] fn test_strh_pre() { - check_bytes("6acd0078", |cb| strh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| strh_pre(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11, #0xc]!"); + assert_snapshot!(cb.hexdump(), @"6acd0078"); } #[test] fn test_strh_post() { - check_bytes("6ac50078", |cb| strh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + let cb = compile(|cb| strh_post(cb, W10, A64Opnd::new_mem(64, X11, 12))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: strh w10, [x11], #0xc"); + assert_snapshot!(cb.hexdump(), @"6ac50078"); } #[test] fn test_stur_64_bits() { - check_bytes("6a0108f8", |cb| stur(cb, X10, A64Opnd::new_mem(64, X11, 128))); + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(64, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur x10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108f8"); } #[test] fn test_stur_32_bits() { - check_bytes("6a0108b8", |cb| stur(cb, X10, A64Opnd::new_mem(32, X11, 128))); + let cb = compile(|cb| stur(cb, X10, A64Opnd::new_mem(32, X11, 128))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: stur w10, [x11, #0x80]"); + assert_snapshot!(cb.hexdump(), @"6a0108b8"); } #[test] fn test_sub_reg() { - check_bytes("200002cb", |cb| sub(cb, X0, X1, X2)); + let cb = compile(|cb| sub(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002cb"); } #[test] fn test_sub_uimm() { - check_bytes("201c00d1", |cb| sub(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); } #[test] fn test_sub_imm_positive() { - check_bytes("201c00d1", |cb| sub(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00d1"); } #[test] fn test_sub_imm_negative() { - check_bytes("201c0091", |cb| sub(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| sub(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c0091"); } #[test] fn test_subs_reg() { - check_bytes("200002eb", |cb| subs(cb, X0, X1, X2)); + let cb = compile(|cb| subs(cb, X0, X1, X2)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, x2"); + assert_snapshot!(cb.hexdump(), @"200002eb"); } #[test] fn test_subs_imm_positive() { - check_bytes("201c00f1", |cb| subs(cb, X0, X1, A64Opnd::new_imm(7))); + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); } #[test] fn test_subs_imm_negative() { - check_bytes("201c00b1", |cb| subs(cb, X0, X1, A64Opnd::new_imm(-7))); + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_imm(-7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00b1"); } #[test] fn test_subs_uimm() { - check_bytes("201c00f1", |cb| subs(cb, X0, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| subs(cb, X0, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: subs x0, x1, #7"); + assert_snapshot!(cb.hexdump(), @"201c00f1"); } #[test] fn test_sxtw() { - check_bytes("6a7d4093", |cb| sxtw(cb, X10, W11)); + let cb = compile(|cb| sxtw(cb, X10, W11)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sxtw x10, w11"); + assert_snapshot!(cb.hexdump(), @"6a7d4093"); } #[test] fn test_tbnz() { - check_bytes("4a005037", |cb| tbnz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + let cb = compile(|cb| tbnz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbnz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005037"); } #[test] fn test_tbz() { - check_bytes("4a005036", |cb| tbz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + let cb = compile(|cb| tbz(cb, X10, A64Opnd::UImm(10), A64Opnd::Imm(2))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tbz w10, #0xa, #8"); + assert_snapshot!(cb.hexdump(), @"4a005036"); } #[test] fn test_tst_register() { - check_bytes("1f0001ea", |cb| tst(cb, X0, X1)); + let cb = compile(|cb| tst(cb, X0, X1)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); } #[test] fn test_tst_immediate() { - check_bytes("3f0840f2", |cb| tst(cb, X1, A64Opnd::new_uimm(7))); + let cb = compile(|cb| tst(cb, X1, A64Opnd::new_uimm(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x1, #7"); + assert_snapshot!(cb.hexdump(), @"3f0840f2"); } #[test] fn test_tst_32b_immediate() { - check_bytes("1f3c0072", |cb| tst(cb, W0, A64Opnd::new_uimm(0xffff))); + let cb = compile(|cb| tst(cb, W0, A64Opnd::new_uimm(0xffff))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0xffff"); + assert_snapshot!(cb.hexdump(), @"1f3c0072"); } #[test] @@ -1756,10 +1977,11 @@ mod tests { add_extended(&mut cb, X30, X30, X30); add_extended(&mut cb, X31, X31, X31); - assert_disasm!(cb, "6a61298bde633e8bff633f8b", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add x10, x11, x9, uxtx 0x4: add x30, x30, x30, uxtx 0x8: add sp, sp, xzr "); + assert_snapshot!(cb.hexdump(), @"6a61298bde633e8bff633f8b"); } } diff --git a/zjit/src/asm/arm64/opnd.rs b/zjit/src/asm/arm64/opnd.rs index a77958f7e6..3e6245826b 100644 --- a/zjit/src/asm/arm64/opnd.rs +++ b/zjit/src/asm/arm64/opnd.rs @@ -1,7 +1,7 @@ - +use std::fmt; /// This operand represents a register. -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Ord, PartialOrd)] pub struct A64Reg { // Size in bits @@ -18,7 +18,7 @@ impl A64Reg { } } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct A64Mem { // Size in bits @@ -42,7 +42,7 @@ impl A64Mem { } } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum A64Opnd { // Dummy operand @@ -79,10 +79,7 @@ impl A64Opnd { /// Convenience function to check if this operand is a register. pub fn is_reg(&self) -> bool { - match self { - A64Opnd::Reg(_) => true, - _ => false - } + matches!(self, A64Opnd::Reg(_)) } /// Unwrap a register from an operand. @@ -197,5 +194,77 @@ pub const W30: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 30 }); pub const W31: A64Opnd = A64Opnd::Reg(A64Reg { num_bits: 32, reg_no: 31 }); // C argument registers -pub const C_ARG_REGS: [A64Opnd; 4] = [X0, X1, X2, X3]; -pub const C_ARG_REGREGS: [A64Reg; 4] = [X0_REG, X1_REG, X2_REG, X3_REG]; +pub const C_ARG_REGS: [A64Opnd; 6] = [X0, X1, X2, X3, X4, X5]; +pub const C_ARG_REGREGS: [A64Reg; 6] = [X0_REG, X1_REG, X2_REG, X3_REG, X4_REG, X5_REG]; + +impl fmt::Display for A64Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match A64Opnd::Reg(*self) { + X0 => write!(f, "x0"), + X1 => write!(f, "x1"), + X2 => write!(f, "x2"), + X3 => write!(f, "x3"), + X4 => write!(f, "x4"), + X5 => write!(f, "x5"), + X6 => write!(f, "x6"), + X7 => write!(f, "x7"), + X8 => write!(f, "x8"), + X9 => write!(f, "x9"), + X10 => write!(f, "x10"), + X11 => write!(f, "x11"), + X12 => write!(f, "x12"), + X13 => write!(f, "x13"), + X14 => write!(f, "x14"), + X15 => write!(f, "x15"), + X16 => write!(f, "x16"), + X17 => write!(f, "x17"), + X18 => write!(f, "x18"), + X19 => write!(f, "x19"), + X20 => write!(f, "x20"), + X21 => write!(f, "x21"), + X22 => write!(f, "x22"), + X23 => write!(f, "x23"), + X24 => write!(f, "x24"), + X25 => write!(f, "x25"), + X26 => write!(f, "x26"), + X27 => write!(f, "x27"), + X28 => write!(f, "x28"), + X29 => write!(f, "x29"), + X30 => write!(f, "x30"), + X31 => write!(f, "x31"), + W0 => write!(f, "w0"), + W1 => write!(f, "w1"), + W2 => write!(f, "w2"), + W3 => write!(f, "w3"), + W4 => write!(f, "w4"), + W5 => write!(f, "w5"), + W6 => write!(f, "w6"), + W7 => write!(f, "w7"), + W8 => write!(f, "w8"), + W9 => write!(f, "w9"), + W10 => write!(f, "w10"), + W11 => write!(f, "w11"), + W12 => write!(f, "w12"), + W13 => write!(f, "w13"), + W14 => write!(f, "w14"), + W15 => write!(f, "w15"), + W16 => write!(f, "w16"), + W17 => write!(f, "w17"), + W18 => write!(f, "w18"), + W19 => write!(f, "w19"), + W20 => write!(f, "w20"), + W21 => write!(f, "w21"), + W22 => write!(f, "w22"), + W23 => write!(f, "w23"), + W24 => write!(f, "w24"), + W25 => write!(f, "w25"), + W26 => write!(f, "w26"), + W27 => write!(f, "w27"), + W28 => write!(f, "w28"), + W29 => write!(f, "w29"), + W30 => write!(f, "w30"), + W31 => write!(f, "w31"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/mod.rs b/zjit/src/asm/mod.rs index 5473998f9d..6583476594 100644 --- a/zjit/src/asm/mod.rs +++ b/zjit/src/asm/mod.rs @@ -1,3 +1,5 @@ +//! Model for creating generating textual assembler code. + use std::collections::BTreeMap; use std::fmt; use std::ops::Range; @@ -14,11 +16,13 @@ pub mod x86_64; pub mod arm64; /// Index to a label created by cb.new_label() -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] pub struct Label(pub usize); +/// The object that knows how to encode the branch instruction. +type BranchEncoder = Box<dyn Fn(&mut CodeBlock, i64, i64) -> Result<(), ()>>; + /// Reference to an ASM label -#[derive(Clone)] pub struct LabelRef { // Position in the code block where the label reference exists pos: usize, @@ -32,7 +36,7 @@ pub struct LabelRef { num_bytes: usize, /// The object that knows how to encode the branch instruction. - encode: fn(&mut CodeBlock, i64, i64) + encode: BranchEncoder, } /// Block of memory into which instructions can be assembled @@ -84,6 +88,11 @@ impl CodeBlock { } } + /// Size of the region in bytes that we have allocated physical memory for. + pub fn mapped_region_size(&self) -> usize { + self.mem_block.borrow().mapped_region_size() + } + /// Add an assembly comment if the feature is on. pub fn add_comment(&mut self, comment: &str) { if !self.keep_comments { @@ -199,6 +208,14 @@ impl CodeBlock { self.dropped_bytes } + /// Set dropped_bytes to false if the current zjit_alloc_bytes() + code_region_size + /// + page_size is below --zjit-mem-size. + pub fn update_dropped_bytes(&mut self) { + if self.mem_block.borrow().can_allocate() { + self.dropped_bytes = false; + } + } + /// Allocate a new label with a given name pub fn new_label(&mut self, name: String) -> Label { assert!(!name.contains(' '), "use underscores in label names, not spaces"); @@ -216,11 +233,11 @@ impl CodeBlock { } // Add a label reference at the current write position - pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: fn(&mut CodeBlock, i64, i64)) { + pub fn label_ref(&mut self, label: Label, num_bytes: usize, encode: impl Fn(&mut CodeBlock, i64, i64) -> Result<(), ()> + 'static) { assert!(label.0 < self.label_addrs.len()); // Keep track of the reference - self.label_refs.push(LabelRef { pos: self.write_pos, label, num_bytes, encode }); + self.label_refs.push(LabelRef { pos: self.write_pos, label, num_bytes, encode: Box::new(encode) }); // Move past however many bytes the instruction takes up if self.write_pos + num_bytes < self.mem_size { @@ -231,8 +248,9 @@ impl CodeBlock { } // Link internal label references - pub fn link_labels(&mut self) { + pub fn link_labels(&mut self) -> Result<(), ()> { let orig_pos = self.write_pos; + let mut link_result = Ok(()); // For each label reference for label_ref in mem::take(&mut self.label_refs) { @@ -244,11 +262,14 @@ impl CodeBlock { assert!(label_addr < self.mem_size); self.write_pos = ref_pos; - (label_ref.encode)(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); + let encode_result = (label_ref.encode.as_ref())(self, (ref_pos + label_ref.num_bytes) as i64, label_addr as i64); + link_result = link_result.and(encode_result); - // Assert that we've written the same number of bytes that we - // expected to have written. - assert!(self.write_pos == ref_pos + label_ref.num_bytes); + // Verify number of bytes written when the callback returns Ok + if encode_result.is_ok() { + assert_eq!(self.write_pos, ref_pos + label_ref.num_bytes, "label_ref \ + callback didn't write number of bytes it claimed to write upfront"); + } } self.write_pos = orig_pos; @@ -257,6 +278,8 @@ impl CodeBlock { self.label_addrs.clear(); self.label_names.clear(); assert!(self.label_refs.is_empty()); + + link_result } /// Convert a Label to CodePtr @@ -271,9 +294,68 @@ impl CodeBlock { } /// Make all the code in the region executable. Call this at the end of a write session. + pub fn mark_all_writable(&mut self) { + self.mem_block.borrow_mut().mark_all_writable(); + } + pub fn mark_all_executable(&mut self) { self.mem_block.borrow_mut().mark_all_executable(); } + + /// Call a func with the disasm of generated code for testing + #[allow(unused_variables)] + #[cfg(all(test, feature = "disasm"))] + pub fn disasm(&self) -> String { + let start_addr = self.get_ptr(0).raw_addr(self); + let end_addr = self.get_write_ptr().raw_addr(self); + crate::disasm::disasm_addr_range(self, start_addr, end_addr) + } + + /// Return the hex dump of generated code for testing + #[cfg(test)] + pub fn hexdump(&self) -> String { + format!("{:x}", self) + } +} + +/// Run assert_snapshot! only if cfg!(feature = "disasm"). +/// $actual can be not only `cb.disasm()` but also `disasms!(cb1, cb2, ...)`. +#[cfg(test)] +#[macro_export] +macro_rules! assert_disasm_snapshot { + ($actual: expr, @$($tt: tt)*) => {{ + #[cfg(feature = "disasm")] + assert_snapshot!($actual, @$($tt)*) + }}; +} + +/// Combine multiple cb.disasm() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms { + ($( $cb:expr ),+ $(,)?) => {{ + crate::disasms_with!("", $( $cb ),+) + }}; +} + +/// Basically `disasms!` but allows a non-"" delimiter, such as "\n" +#[cfg(all(test, feature = "disasm"))] +#[macro_export] +macro_rules! disasms_with { + ($join:expr, $( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.disasm() ),+].join($join) + }}; +} + +/// Combine multiple cb.hexdump() results to match all of them at once, which allows +/// us to avoid running the set of zjit-test -> zjit-test-update multiple times. +#[cfg(test)] +#[macro_export] +macro_rules! hexdumps { + ($( $cb:expr ),+ $(,)?) => {{ + vec![$( $cb.hexdump() ),+].join("\n") + }}; } /// Produce hex string output from the bytes in a code block @@ -282,7 +364,7 @@ impl fmt::LowerHex for CodeBlock { for pos in 0..self.write_pos { let mem_block = &*self.mem_block.borrow(); let byte = unsafe { mem_block.start_ptr().raw_ptr(mem_block).add(pos).read() }; - fmtr.write_fmt(format_args!("{:02x}", byte))?; + fmtr.write_fmt(format_args!("{byte:02x}"))?; } Ok(()) } @@ -292,15 +374,13 @@ impl fmt::LowerHex for CodeBlock { impl CodeBlock { /// Stubbed CodeBlock for testing. Can't execute generated code. pub fn new_dummy() -> Self { - use std::ptr::NonNull; - use crate::virtualmem::*; - use crate::virtualmem::tests::TestingAllocator; - - let mem_size = 1024; - let alloc = TestingAllocator::new(mem_size); - let mem_start: *const u8 = alloc.mem_start(); - let virt_mem = VirtualMem::new(alloc, 1, NonNull::new(mem_start as *mut u8).unwrap(), mem_size, 128 * 1024 * 1024); + const DEFAULT_MEM_SIZE: usize = 1024 * 1024; + CodeBlock::new_dummy_sized(DEFAULT_MEM_SIZE) + } + pub fn new_dummy_sized(mem_size: usize) -> Self { + use crate::virtualmem::*; + let virt_mem = VirtualMem::alloc(mem_size, None); Self::new(Rc::new(RefCell::new(virt_mem)), false) } } @@ -325,7 +405,7 @@ pub fn imm_num_bits(imm: i64) -> u8 return 32; } - return 64; + 64 } /// Compute the number of bits needed to encode an unsigned value @@ -342,7 +422,7 @@ pub fn uimm_num_bits(uimm: u64) -> u8 return 32; } - return 64; + 64 } #[cfg(test)] @@ -381,4 +461,3 @@ mod tests assert_eq!(uimm_num_bits(u64::MAX), 64); } } - diff --git a/zjit/src/asm/x86_64/mod.rs b/zjit/src/asm/x86_64/mod.rs index f1eaa49f20..c2733e783f 100644 --- a/zjit/src/asm/x86_64/mod.rs +++ b/zjit/src/asm/x86_64/mod.rs @@ -25,7 +25,7 @@ pub struct X86UImm pub value: u64 } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum RegType { GP, @@ -34,7 +34,7 @@ pub enum RegType IP, } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct X86Reg { // Size in bits @@ -47,7 +47,7 @@ pub struct X86Reg pub reg_no: u8, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct X86Mem { // Size in bits @@ -66,7 +66,7 @@ pub struct X86Mem pub disp: i32, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq)] pub enum X86Opnd { // Dummy operand @@ -102,6 +102,10 @@ impl X86Reg { reg_no: self.reg_no } } + + pub fn rex_needed(&self) -> bool { + self.reg_no > 7 || self.num_bits == 8 && self.reg_no >= 4 + } } impl X86Opnd { @@ -110,7 +114,7 @@ impl X86Opnd { X86Opnd::None => false, X86Opnd::Imm(_) => false, X86Opnd::UImm(_) => false, - X86Opnd::Reg(reg) => reg.reg_no > 7 || reg.num_bits == 8 && reg.reg_no >= 4, + X86Opnd::Reg(reg) => reg.rex_needed(), X86Opnd::Mem(mem) => mem.base_reg_no > 7 || (mem.idx_reg_no.unwrap_or(0) > 7), X86Opnd::IPRel(_) => false } @@ -163,10 +167,7 @@ impl X86Opnd { } pub fn is_some(&self) -> bool { - match self { - X86Opnd::None => false, - _ => true - } + !matches!(self, X86Opnd::None) } } @@ -284,11 +285,11 @@ pub fn mem_opnd(num_bits: u8, base_reg: X86Opnd, disp: i32) -> X86Opnd } else { X86Opnd::Mem( X86Mem { - num_bits: num_bits, + num_bits, base_reg_no: base_reg.reg_no, idx_reg_no: None, scale_exp: 0, - disp: disp, + disp, } ) } @@ -512,7 +513,7 @@ fn write_rm_unary(cb: &mut CodeBlock, op_mem_reg_8: u8, op_mem_reg_pref: u8, op_ // Encode an add-like RM instruction with multiple possible encodings fn write_rm_multi(cb: &mut CodeBlock, op_mem_reg8: u8, op_mem_reg_pref: u8, op_reg_mem8: u8, op_reg_mem_pref: u8, op_mem_imm8: u8, op_mem_imm_sml: u8, op_mem_imm_lrg: u8, op_ext_imm: Option<u8>, opnd0: X86Opnd, opnd1: X86Opnd) { - assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_))); + assert!(matches!(opnd0, X86Opnd::Reg(_) | X86Opnd::Mem(_)), "unexpected opnd0: {opnd0:?}, {opnd1:?}"); // Check the size of opnd0 let opnd_size = opnd0.num_bits(); @@ -682,6 +683,7 @@ pub fn call_label(cb: &mut CodeBlock, label: Label) { cb.label_ref(label, 5, |cb, src_addr, dst_addr| { cb.write_byte(0xE8); cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) }); } @@ -782,13 +784,6 @@ pub fn imul(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { write_rm(cb, false, true, opnd0, opnd1, None, &[0x0F, 0xAF]); } - // Flip the operands to handle this case. This instruction has weird encoding restrictions. - (X86Opnd::Mem(_), X86Opnd::Reg(_)) => { - //REX.W + 0F AF /rIMUL r64, r/m64 - // Quadword register := Quadword register * r/m64. - write_rm(cb, false, true, opnd1, opnd0, None, &[0x0F, 0xAF]); - } - _ => unreachable!() } } @@ -805,6 +800,7 @@ fn write_jcc<const OP: u8>(cb: &mut CodeBlock, label: Label) { cb.write_byte(0x0F); cb.write_byte(OP); cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) }); } @@ -844,6 +840,7 @@ pub fn jmp_label(cb: &mut CodeBlock, label: Label) { cb.label_ref(label, 5, |cb, src_addr, dst_addr| { cb.write_byte(0xE9); cb.write_int((dst_addr - src_addr) as u64, 32); + Ok(()) }); } @@ -930,60 +927,43 @@ pub fn lea(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { /// mov - Data move operation pub fn mov(cb: &mut CodeBlock, dst: X86Opnd, src: X86Opnd) { - match (dst, src) { - // R + Imm - (X86Opnd::Reg(reg), X86Opnd::Imm(imm)) => { - assert!(imm.num_bits <= reg.num_bits); - - // In case the source immediate could be zero extended to be 64 - // bit, we can use the 32-bit operands version of the instruction. - // For example, we can turn mov(rax, 0x34) into the equivalent - // mov(eax, 0x34). - if (reg.num_bits == 64) && (imm.value > 0) && (imm.num_bits <= 32) { - if dst.rex_needed() { - write_rex(cb, false, 0, 0, reg.reg_no); - } - write_opcode(cb, 0xB8, reg); - cb.write_int(imm.value as u64, 32); - } else { - if reg.num_bits == 16 { - cb.write_byte(0x66); - } - - if dst.rex_needed() || reg.num_bits == 64 { - write_rex(cb, reg.num_bits == 64, 0, 0, reg.reg_no); - } + fn emit_reg_imm(cb: &mut CodeBlock, reg: X86Reg, imm: X86Imm) { + // In case the source immediate could be zero extended to be 64 + // bit, we can use the 32-bit operands version of the instruction. + // For example, we can turn mov(rax, 0x34) into the equivalent + // mov(eax, 0x34). + if (reg.num_bits == 64) && u32::try_from(imm.value).is_ok() { + if reg.rex_needed() { + write_rex(cb, false, 0, 0, reg.reg_no); + } + write_opcode(cb, 0xB8, reg); + cb.write_int(imm.value as u64, 32); + } else if reg.num_bits == 64 && imm.num_bits <= 32 { + // Use 32-to-64 bit sign-extension when possible + write_rm(cb, false, true /* REX.w */, X86Opnd::None, X86Opnd::Reg(reg), Some(0) /* /0 */, &[0xc7]); + cb.write_int(imm.value as u64, 32); + } else { + if reg.num_bits == 16 { + cb.write_byte(0x66); + } - write_opcode(cb, if reg.num_bits == 8 { 0xb0 } else { 0xb8 }, reg); - cb.write_int(imm.value as u64, reg.num_bits.into()); + if reg.rex_needed() || reg.num_bits == 64 { + write_rex(cb, reg.num_bits == 64, 0, 0, reg.reg_no); } - }, + + write_opcode(cb, if reg.num_bits == 8 { 0xb0 } else { 0xb8 }, reg); + cb.write_int(imm.value as u64, reg.num_bits.into()); + } + } + match (dst, src) { + // R + Imm + (X86Opnd::Reg(reg), X86Opnd::Imm(imm)) => emit_reg_imm(cb, reg, imm), // R + UImm (X86Opnd::Reg(reg), X86Opnd::UImm(uimm)) => { - assert!(uimm.num_bits <= reg.num_bits); - - // In case the source immediate could be zero extended to be 64 - // bit, we can use the 32-bit operands version of the instruction. - // For example, we can turn mov(rax, 0x34) into the equivalent - // mov(eax, 0x34). - if (reg.num_bits == 64) && (uimm.value <= u32::MAX.into()) { - if dst.rex_needed() { - write_rex(cb, false, 0, 0, reg.reg_no); - } - write_opcode(cb, 0xB8, reg); - cb.write_int(uimm.value, 32); - } else { - if reg.num_bits == 16 { - cb.write_byte(0x66); - } - - if dst.rex_needed() || reg.num_bits == 64 { - write_rex(cb, reg.num_bits == 64, 0, 0, reg.reg_no); - } - - write_opcode(cb, if reg.num_bits == 8 { 0xb0 } else { 0xb8 }, reg); - cb.write_int(uimm.value, reg.num_bits.into()); - } + // u64->i64 type cast is a bit pattern no-op + let value: u64 = uimm.value; + let value = value as i64; + emit_reg_imm(cb, reg, X86Imm { num_bits: imm_num_bits(value), value }); }, // M + Imm (X86Opnd::Mem(mem), X86Opnd::Imm(imm)) => { @@ -1165,6 +1145,9 @@ pub fn push(cb: &mut CodeBlock, opnd: X86Opnd) { X86Opnd::Mem(_mem) => { write_rm(cb, false, false, X86Opnd::None, opnd, Some(6), &[0xff]); }, + X86Opnd::Imm(X86Imm { value: 0, .. }) | X86Opnd::UImm(X86UImm { value: 0, .. }) => { + cb.write_bytes(&[0x6a, 0x00]); + } _ => unreachable!() } } @@ -1337,7 +1320,7 @@ pub fn test(cb: &mut CodeBlock, rm_opnd: X86Opnd, test_opnd: X86Opnd) { write_rm(cb, rm_num_bits == 16, rm_num_bits == 64, test_opnd, rm_opnd, None, &[0x85]); } }, - _ => unreachable!() + _ => unreachable!("unexpected operands for test: {rm_opnd:?}, {test_opnd:?}") }; } @@ -1383,3 +1366,74 @@ pub fn xor(cb: &mut CodeBlock, opnd0: X86Opnd, opnd1: X86Opnd) { opnd1 ); } + +impl fmt::Display for X86Reg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match X86Opnd::Reg(*self) { + RAX => write!(f, "rax"), + RCX => write!(f, "rcx"), + RDX => write!(f, "rdx"), + RBX => write!(f, "rbx"), + RSP => write!(f, "rsp"), + RBP => write!(f, "rbp"), + RSI => write!(f, "rsi"), + RDI => write!(f, "rdi"), + R8 => write!(f, "r8"), + R9 => write!(f, "r9"), + R10 => write!(f, "r10"), + R11 => write!(f, "r11"), + R12 => write!(f, "r12"), + R13 => write!(f, "r13"), + R14 => write!(f, "r14"), + R15 => write!(f, "r15"), + EAX => write!(f, "eax"), + ECX => write!(f, "ecx"), + EDX => write!(f, "edx"), + EBX => write!(f, "ebx"), + ESP => write!(f, "esp"), + EBP => write!(f, "ebp"), + ESI => write!(f, "esi"), + EDI => write!(f, "edi"), + R8D => write!(f, "r8d"), + R9D => write!(f, "r9d"), + R10D => write!(f, "r10d"), + R11D => write!(f, "r11d"), + R12D => write!(f, "r12d"), + R13D => write!(f, "r13d"), + R14D => write!(f, "r14d"), + R15D => write!(f, "r15d"), + AX => write!(f, "ax"), + CX => write!(f, "cx"), + DX => write!(f, "dx"), + BX => write!(f, "bx"), + BP => write!(f, "bp"), + SI => write!(f, "si"), + DI => write!(f, "di"), + R8W => write!(f, "r8w"), + R9W => write!(f, "r9w"), + R10W => write!(f, "r10w"), + R11W => write!(f, "r11w"), + R12W => write!(f, "r12w"), + R13W => write!(f, "r13w"), + R14W => write!(f, "r14w"), + R15W => write!(f, "r15w"), + AL => write!(f, "al"), + CL => write!(f, "cl"), + DL => write!(f, "dl"), + BL => write!(f, "bl"), + SPL => write!(f, "spl"), + BPL => write!(f, "bpl"), + SIL => write!(f, "sil"), + DIL => write!(f, "dil"), + R8B => write!(f, "r8b"), + R9B => write!(f, "r9b"), + R10B => write!(f, "r10b"), + R11B => write!(f, "r11b"), + R12B => write!(f, "r12b"), + R13B => write!(f, "r13b"), + R14B => write!(f, "r14b"), + R15B => write!(f, "r15b"), + _ => write!(f, "{self:?}"), + } + } +} diff --git a/zjit/src/asm/x86_64/tests.rs b/zjit/src/asm/x86_64/tests.rs index ec490fd330..0dfee26496 100644 --- a/zjit/src/asm/x86_64/tests.rs +++ b/zjit/src/asm/x86_64/tests.rs @@ -1,6 +1,10 @@ #![cfg(test)] -use crate::asm::x86_64::*; +use insta::assert_snapshot; + +#[cfg(feature = "disasm")] +use crate::disasms; +use crate::{asm::x86_64::*, hexdumps, assert_disasm_snapshot}; /// Check that the bytes for an instruction sequence match a hex string fn check_bytes<R>(bytes: &str, run: R) where R: FnOnce(&mut super::CodeBlock) { @@ -9,364 +13,828 @@ fn check_bytes<R>(bytes: &str, run: R) where R: FnOnce(&mut super::CodeBlock) { assert_eq!(format!("{:x}", cb), bytes); } +fn compile<R>(run: R) -> CodeBlock where R: FnOnce(&mut super::CodeBlock) { + let mut cb = super::CodeBlock::new_dummy(); + run(&mut cb); + cb +} + #[test] fn test_add() { - check_bytes("80c103", |cb| add(cb, CL, imm_opnd(3))); - check_bytes("00d9", |cb| add(cb, CL, BL)); - check_bytes("4000e1", |cb| add(cb, CL, SPL)); - check_bytes("6601d9", |cb| add(cb, CX, BX)); - check_bytes("4801d8", |cb| add(cb, RAX, RBX)); - check_bytes("01d1", |cb| add(cb, ECX, EDX)); - check_bytes("4c01f2", |cb| add(cb, RDX, R14)); - check_bytes("480110", |cb| add(cb, mem_opnd(64, RAX, 0), RDX)); - check_bytes("480310", |cb| add(cb, RDX, mem_opnd(64, RAX, 0))); - check_bytes("48035008", |cb| add(cb, RDX, mem_opnd(64, RAX, 8))); - check_bytes("480390ff000000", |cb| add(cb, RDX, mem_opnd(64, RAX, 255))); - check_bytes("4881407fff000000", |cb| add(cb, mem_opnd(64, RAX, 127), imm_opnd(255))); - check_bytes("0110", |cb| add(cb, mem_opnd(32, RAX, 0), EDX)); - check_bytes("4883c408", |cb| add(cb, RSP, imm_opnd(8))); - check_bytes("83c108", |cb| add(cb, ECX, imm_opnd(8))); - check_bytes("81c1ff000000", |cb| add(cb, ECX, imm_opnd(255))); + let cb01 = compile(|cb| add(cb, CL, imm_opnd(3))); + let cb02 = compile(|cb| add(cb, CL, BL)); + let cb03 = compile(|cb| add(cb, CL, SPL)); + let cb04 = compile(|cb| add(cb, CX, BX)); + let cb05 = compile(|cb| add(cb, RAX, RBX)); + let cb06 = compile(|cb| add(cb, ECX, EDX)); + let cb07 = compile(|cb| add(cb, RDX, R14)); + let cb08 = compile(|cb| add(cb, mem_opnd(64, RAX, 0), RDX)); + let cb09 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 0))); + let cb10 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 8))); + let cb11 = compile(|cb| add(cb, RDX, mem_opnd(64, RAX, 255))); + let cb12 = compile(|cb| add(cb, mem_opnd(64, RAX, 127), imm_opnd(255))); + let cb13 = compile(|cb| add(cb, mem_opnd(32, RAX, 0), EDX)); + let cb14 = compile(|cb| add(cb, RSP, imm_opnd(8))); + let cb15 = compile(|cb| add(cb, ECX, imm_opnd(8))); + let cb16 = compile(|cb| add(cb, ECX, imm_opnd(255))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @" + 0x0: add cl, 3 + 0x0: add cl, bl + 0x0: add cl, spl + 0x0: add cx, bx + 0x0: add rax, rbx + 0x0: add ecx, edx + 0x0: add rdx, r14 + 0x0: add qword ptr [rax], rdx + 0x0: add rdx, qword ptr [rax] + 0x0: add rdx, qword ptr [rax + 8] + 0x0: add rdx, qword ptr [rax + 0xff] + 0x0: add qword ptr [rax + 0x7f], 0xff + 0x0: add dword ptr [rax], edx + 0x0: add rsp, 8 + 0x0: add ecx, 8 + 0x0: add ecx, 0xff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16), @" + 80c103 + 00d9 + 4000e1 + 6601d9 + 4801d8 + 01d1 + 4c01f2 + 480110 + 480310 + 48035008 + 480390ff000000 + 4881407fff000000 + 0110 + 4883c408 + 83c108 + 81c1ff000000 + "); } #[test] fn test_add_unsigned() { // ADD r/m8, imm8 - check_bytes("4180c001", |cb| add(cb, R8B, uimm_opnd(1))); - check_bytes("4180c07f", |cb| add(cb, R8B, imm_opnd(i8::MAX.try_into().unwrap()))); - + let cb1 = compile(|cb| add(cb, R8B, uimm_opnd(1))); + let cb2 = compile(|cb| add(cb, R8B, imm_opnd(i8::MAX.into()))); // ADD r/m16, imm16 - check_bytes("664183c001", |cb| add(cb, R8W, uimm_opnd(1))); - check_bytes("664181c0ff7f", |cb| add(cb, R8W, uimm_opnd(i16::MAX.try_into().unwrap()))); - + let cb3 = compile(|cb| add(cb, R8W, uimm_opnd(1))); + let cb4 = compile(|cb| add(cb, R8W, uimm_opnd(i16::MAX.try_into().unwrap()))); // ADD r/m32, imm32 - check_bytes("4183c001", |cb| add(cb, R8D, uimm_opnd(1))); - check_bytes("4181c0ffffff7f", |cb| add(cb, R8D, uimm_opnd(i32::MAX.try_into().unwrap()))); - + let cb5 = compile(|cb| add(cb, R8D, uimm_opnd(1))); + let cb6 = compile(|cb| add(cb, R8D, uimm_opnd(i32::MAX.try_into().unwrap()))); // ADD r/m64, imm32 - check_bytes("4983c001", |cb| add(cb, R8, uimm_opnd(1))); - check_bytes("4981c0ffffff7f", |cb| add(cb, R8, uimm_opnd(i32::MAX.try_into().unwrap()))); + let cb7 = compile(|cb| add(cb, R8, uimm_opnd(1))); + let cb8 = compile(|cb| add(cb, R8, uimm_opnd(i32::MAX.try_into().unwrap()))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 0x0: add r8b, 1 + 0x0: add r8b, 0x7f + 0x0: add r8w, 1 + 0x0: add r8w, 0x7fff + 0x0: add r8d, 1 + 0x0: add r8d, 0x7fffffff + 0x0: add r8, 1 + 0x0: add r8, 0x7fffffff + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 4180c001 + 4180c07f + 664183c001 + 664181c0ff7f + 4183c001 + 4181c0ffffff7f + 4983c001 + 4981c0ffffff7f + "); } #[test] fn test_and() { - check_bytes("4421e5", |cb| and(cb, EBP, R12D)); - check_bytes("48832008", |cb| and(cb, mem_opnd(64, RAX, 0), imm_opnd(0x08))); + let cb1 = compile(|cb| and(cb, EBP, R12D)); + let cb2 = compile(|cb| and(cb, mem_opnd(64, RAX, 0), imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: and ebp, r12d + 0x0: and qword ptr [rax], 8 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + 4421e5 + 48832008 + "); } #[test] fn test_call_label() { - check_bytes("e8fbffffff", |cb| { + let cb = compile(|cb| { let label_idx = cb.new_label("fn".to_owned()); call_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); } #[test] fn test_call_ptr() { // calling a lower address - check_bytes("e8fbffffff", |cb| { + let cb = compile(|cb| { let ptr = cb.get_write_ptr(); call_ptr(cb, RAX, ptr.raw_ptr(cb)); }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call 0"); + assert_snapshot!(cb.hexdump(), @"e8fbffffff"); } #[test] fn test_call_reg() { - check_bytes("ffd0", |cb| call(cb, RAX)); + let cb = compile(|cb| call(cb, RAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call rax"); + assert_snapshot!(cb.hexdump(), @"ffd0"); } #[test] fn test_call_mem() { - check_bytes("ff542408", |cb| call(cb, mem_opnd(64, RSP, 8))); + let cb = compile(|cb| call(cb, mem_opnd(64, RSP, 8))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: call qword ptr [rsp + 8]"); + assert_snapshot!(cb.hexdump(), @"ff542408"); } #[test] fn test_cmovcc() { - check_bytes("0f4ff7", |cb| cmovg(cb, ESI, EDI)); - check_bytes("0f4f750c", |cb| cmovg(cb, ESI, mem_opnd(32, RBP, 12))); - check_bytes("0f4cc1", |cb| cmovl(cb, EAX, ECX)); - check_bytes("480f4cdd", |cb| cmovl(cb, RBX, RBP)); - check_bytes("0f4e742404", |cb| cmovle(cb, ESI, mem_opnd(32, RSP, 4))); + let cb1 = compile(|cb| cmovg(cb, ESI, EDI)); + let cb2 = compile(|cb| cmovg(cb, ESI, mem_opnd(32, RBP, 12))); + let cb3 = compile(|cb| cmovl(cb, EAX, ECX)); + let cb4 = compile(|cb| cmovl(cb, RBX, RBP)); + let cb5 = compile(|cb| cmovle(cb, ESI, mem_opnd(32, RSP, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: cmovg esi, edi + 0x0: cmovg esi, dword ptr [rbp + 0xc] + 0x0: cmovl eax, ecx + 0x0: cmovl rbx, rbp + 0x0: cmovle esi, dword ptr [rsp + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 0f4ff7 + 0f4f750c + 0f4cc1 + 480f4cdd + 0f4e742404 + "); } #[test] fn test_cmp() { - check_bytes("38d1", |cb| cmp(cb, CL, DL)); - check_bytes("39f9", |cb| cmp(cb, ECX, EDI)); - check_bytes("493b1424", |cb| cmp(cb, RDX, mem_opnd(64, R12, 0))); - check_bytes("4883f802", |cb| cmp(cb, RAX, imm_opnd(2))); - check_bytes("81f900000080", |cb| cmp(cb, ECX, uimm_opnd(0x8000_0000))); + let cb1 = compile(|cb| cmp(cb, CL, DL)); + let cb2 = compile(|cb| cmp(cb, ECX, EDI)); + let cb3 = compile(|cb| cmp(cb, RDX, mem_opnd(64, R12, 0))); + let cb4 = compile(|cb| cmp(cb, RAX, imm_opnd(2))); + let cb5 = compile(|cb| cmp(cb, ECX, uimm_opnd(0x8000_0000))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: cmp cl, dl + 0x0: cmp ecx, edi + 0x0: cmp rdx, qword ptr [r12] + 0x0: cmp rax, 2 + 0x0: cmp ecx, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 38d1 + 39f9 + 493b1424 + 4883f802 + 81f900000080 + "); } #[test] fn test_cqo() { - check_bytes("4899", |cb| cqo(cb)); + let cb = compile(cqo); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cqo"); + assert_snapshot!(cb.hexdump(), @"4899"); } #[test] fn test_imul() { - check_bytes("480fafc3", |cb| imul(cb, RAX, RBX)); - check_bytes("480faf10", |cb| imul(cb, RDX, mem_opnd(64, RAX, 0))); + let cb1 = compile(|cb| imul(cb, RAX, RBX)); + let cb2 = compile(|cb| imul(cb, RDX, mem_opnd(64, RAX, 0))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: imul rax, rbx + 0x0: imul rdx, qword ptr [rax] + "); - // Operands flipped for encoding since multiplication is commutative - check_bytes("480faf10", |cb| imul(cb, mem_opnd(64, RAX, 0), RDX)); + assert_snapshot!(hexdumps!(cb1, cb2), @" + 480fafc3 + 480faf10 + "); +} + +#[test] +#[should_panic] +fn test_imul_mem_reg() { + // imul doesn't have (Mem, Reg) encoding. Since multiplication is communicative, imul() could + // swap operands. However, x86_scratch_split may need to move the result to the output operand, + // which can be complicated if the assembler may sometimes change the result operand. + // So x86_scratch_split should be responsible for that swap, not the assembler. + compile(|cb| imul(cb, mem_opnd(64, RAX, 0), RDX)); } #[test] fn test_jge_label() { - check_bytes("0f8dfaffffff", |cb| { + let cb = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); jge_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jge 0"); + assert_snapshot!(cb.hexdump(), @"0f8dfaffffff"); } #[test] fn test_jmp_label() { // Forward jump - check_bytes("e900000000", |cb| { + let cb1 = compile(|cb| { let label_idx = cb.new_label("next".to_owned()); jmp_label(cb, label_idx); cb.write_label(label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); - // Backwards jump - check_bytes("e9fbffffff", |cb| { + let cb2 = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); cb.write_label(label_idx); jmp_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: jmp 5 + 0x0: jmp 0 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + e900000000 + e9fbffffff + "); } #[test] fn test_jmp_rm() { - check_bytes("41ffe4", |cb| jmp_rm(cb, R12)); + let cb = compile(|cb| jmp_rm(cb, R12)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jmp r12"); + assert_snapshot!(cb.hexdump(), @"41ffe4"); } #[test] fn test_jo_label() { - check_bytes("0f80faffffff", |cb| { + let cb = compile(|cb| { let label_idx = cb.new_label("loop".to_owned()); jo_label(cb, label_idx); - cb.link_labels(); + cb.link_labels().unwrap(); }); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: jo 0"); + assert_snapshot!(cb.hexdump(), @"0f80faffffff"); } #[test] fn test_lea() { - check_bytes("488d5108", |cb| lea(cb, RDX, mem_opnd(64, RCX, 8))); - check_bytes("488d0500000000", |cb| lea(cb, RAX, mem_opnd(8, RIP, 0))); - check_bytes("488d0505000000", |cb| lea(cb, RAX, mem_opnd(8, RIP, 5))); - check_bytes("488d3d05000000", |cb| lea(cb, RDI, mem_opnd(8, RIP, 5))); + let cb1 = compile(|cb| lea(cb, RDX, mem_opnd(64, RCX, 8))); + let cb2 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 0))); + let cb3 = compile(|cb| lea(cb, RAX, mem_opnd(8, RIP, 5))); + let cb4 = compile(|cb| lea(cb, RDI, mem_opnd(8, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @" + 0x0: lea rdx, [rcx + 8] + 0x0: lea rax, [rip] + 0x0: lea rax, [rip + 5] + 0x0: lea rdi, [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @" + 488d5108 + 488d0500000000 + 488d0505000000 + 488d3d05000000 + "); } #[test] fn test_mov() { - check_bytes("b807000000", |cb| mov(cb, EAX, imm_opnd(7))); - check_bytes("b8fdffffff", |cb| mov(cb, EAX, imm_opnd(-3))); - check_bytes("41bf03000000", |cb| mov(cb, R15, imm_opnd(3))); - check_bytes("89d8", |cb| mov(cb, EAX, EBX)); - check_bytes("89c8", |cb| mov(cb, EAX, ECX)); - check_bytes("8b9380000000", |cb| mov(cb, EDX, mem_opnd(32, RBX, 128))); - check_bytes("488b442404", |cb| mov(cb, RAX, mem_opnd(64, RSP, 4))); - + let cb01 = compile(|cb| mov(cb, EAX, imm_opnd(7))); + let cb02 = compile(|cb| mov(cb, EAX, imm_opnd(-3))); + let cb03 = compile(|cb| mov(cb, R15, imm_opnd(3))); + let cb04 = compile(|cb| mov(cb, EAX, EBX)); + let cb05 = compile(|cb| mov(cb, EAX, ECX)); + let cb06 = compile(|cb| mov(cb, EDX, mem_opnd(32, RBX, 128))); + let cb07 = compile(|cb| mov(cb, RAX, mem_opnd(64, RSP, 4))); // Test `mov rax, 3` => `mov eax, 3` optimization - check_bytes("41b834000000", |cb| mov(cb, R8, imm_opnd(0x34))); - check_bytes("49b80000008000000000", |cb| mov(cb, R8, imm_opnd(0x80000000))); - check_bytes("49b8ffffffffffffffff", |cb| mov(cb, R8, imm_opnd(-1))); - - check_bytes("b834000000", |cb| mov(cb, RAX, imm_opnd(0x34))); - check_bytes("48b8020000000000c0ff", |cb| mov(cb, RAX, imm_opnd(-18014398509481982))); - check_bytes("48b80000008000000000", |cb| mov(cb, RAX, imm_opnd(0x80000000))); - check_bytes("48b8ccffffffffffffff", |cb| mov(cb, RAX, imm_opnd(-52))); // yasm thinks this could use a dword immediate instead of qword - check_bytes("48b8ffffffffffffffff", |cb| mov(cb, RAX, imm_opnd(-1))); // yasm thinks this could use a dword immediate instead of qword - check_bytes("4488c9", |cb| mov(cb, CL, R9B)); - check_bytes("4889c3", |cb| mov(cb, RBX, RAX)); - check_bytes("4889df", |cb| mov(cb, RDI, RBX)); - check_bytes("40b60b", |cb| mov(cb, SIL, imm_opnd(11))); - - check_bytes("c60424fd", |cb| mov(cb, mem_opnd(8, RSP, 0), imm_opnd(-3))); - check_bytes("48c7470801000000", |cb| mov(cb, mem_opnd(64, RDI, 8), imm_opnd(1))); - //check_bytes("67c7400411000000", |cb| mov(cb, mem_opnd(32, EAX, 4), imm_opnd(0x34))); // We don't distinguish between EAX and RAX here - that's probably fine? - check_bytes("c7400411000000", |cb| mov(cb, mem_opnd(32, RAX, 4), imm_opnd(17))); - check_bytes("c7400401000080", |cb| mov(cb, mem_opnd(32, RAX, 4), uimm_opnd(0x80000001))); - check_bytes("41895814", |cb| mov(cb, mem_opnd(32, R8, 20), EBX)); - check_bytes("4d8913", |cb| mov(cb, mem_opnd(64, R11, 0), R10)); - check_bytes("48c742f8f4ffffff", |cb| mov(cb, mem_opnd(64, RDX, -8), imm_opnd(-12))); + let cb08 = compile(|cb| mov(cb, R8, imm_opnd(0x34))); + let cb09 = compile(|cb| mov(cb, R8, imm_opnd(0x80000000))); + let cb10 = compile(|cb| mov(cb, R8, imm_opnd(-1))); + let cb11 = compile(|cb| mov(cb, RAX, imm_opnd(0x34))); + let cb12 = compile(|cb| mov(cb, RAX, imm_opnd(-18014398509481982))); + let cb13 = compile(|cb| mov(cb, RAX, imm_opnd(0x80000000))); + let cb14 = compile(|cb| mov(cb, RAX, imm_opnd(-52))); // yasm thinks this could use a dword immediate instead of qword + let cb15 = compile(|cb| mov(cb, RAX, imm_opnd(-1))); // yasm thinks this could use a dword immediate instead of qword + let cb16 = compile(|cb| mov(cb, CL, R9B)); + let cb17 = compile(|cb| mov(cb, RBX, RAX)); + let cb18 = compile(|cb| mov(cb, RDI, RBX)); + let cb19 = compile(|cb| mov(cb, SIL, imm_opnd(11))); + let cb20 = compile(|cb| mov(cb, mem_opnd(8, RSP, 0), imm_opnd(-3))); + let cb21 = compile(|cb| mov(cb, mem_opnd(64, RDI, 8), imm_opnd(1))); + //let cb = compile(|cb| mov(cb, mem_opnd(32, EAX, 4), imm_opnd(0x34))); // We don't distinguish between EAX and RAX here - that's probably fine? + let cb22 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), imm_opnd(17))); + let cb23 = compile(|cb| mov(cb, mem_opnd(32, RAX, 4), uimm_opnd(0x80000001))); + let cb24 = compile(|cb| mov(cb, mem_opnd(32, R8, 20), EBX)); + let cb25 = compile(|cb| mov(cb, mem_opnd(64, R11, 0), R10)); + let cb26 = compile(|cb| mov(cb, mem_opnd(64, RDX, -8), imm_opnd(-12))); + + assert_disasm_snapshot!(disasms!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26, + ), @" + 0x0: mov eax, 7 + 0x0: mov eax, 0xfffffffd + 0x0: mov r15d, 3 + 0x0: mov eax, ebx + 0x0: mov eax, ecx + 0x0: mov edx, dword ptr [rbx + 0x80] + 0x0: mov rax, qword ptr [rsp + 4] + 0x0: mov r8d, 0x34 + 0x0: mov r8d, 0x80000000 + 0x0: mov r8, 0xffffffffffffffff + 0x0: mov eax, 0x34 + 0x0: movabs rax, 0xffc0000000000002 + 0x0: mov eax, 0x80000000 + 0x0: mov rax, 0xffffffffffffffcc + 0x0: mov rax, 0xffffffffffffffff + 0x0: mov cl, r9b + 0x0: mov rbx, rax + 0x0: mov rdi, rbx + 0x0: mov sil, 0xb + 0x0: mov byte ptr [rsp], 0xfd + 0x0: mov qword ptr [rdi + 8], 1 + 0x0: mov dword ptr [rax + 4], 0x11 + 0x0: mov dword ptr [rax + 4], 0x80000001 + 0x0: mov dword ptr [r8 + 0x14], ebx + 0x0: mov qword ptr [r11], r10 + 0x0: mov qword ptr [rdx - 8], 0xfffffffffffffff4 + "); + + assert_snapshot!(hexdumps!( + cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, + cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21, cb22, cb23, cb24, cb25, cb26 + ), @" + b807000000 + b8fdffffff + 41bf03000000 + 89d8 + 89c8 + 8b9380000000 + 488b442404 + 41b834000000 + 41b800000080 + 49c7c0ffffffff + b834000000 + 48b8020000000000c0ff + b800000080 + 48c7c0ccffffff + 48c7c0ffffffff + 4488c9 + 4889c3 + 4889df + 40b60b + c60424fd + 48c7470801000000 + c7400411000000 + c7400401000080 + 41895814 + 4d8913 + 48c742f8f4ffffff + "); } #[test] fn test_movabs() { - check_bytes("49b83400000000000000", |cb| movabs(cb, R8, 0x34)); - check_bytes("49b80000008000000000", |cb| movabs(cb, R8, 0x80000000)); + let cb1 = compile(|cb| movabs(cb, R8, 0x34)); + let cb2 = compile(|cb| movabs(cb, R8, 0x80000000)); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: movabs r8, 0x34 + 0x0: movabs r8, 0x80000000 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + 49b83400000000000000 + 49b80000008000000000 + "); } #[test] fn test_mov_unsigned() { // MOV AL, imm8 - check_bytes("b001", |cb| mov(cb, AL, uimm_opnd(1))); - check_bytes("b0ff", |cb| mov(cb, AL, uimm_opnd(u8::MAX.into()))); - + let cb01 = compile(|cb| mov(cb, AL, uimm_opnd(1))); + let cb02 = compile(|cb| mov(cb, AL, uimm_opnd(u8::MAX.into()))); // MOV AX, imm16 - check_bytes("66b80100", |cb| mov(cb, AX, uimm_opnd(1))); - check_bytes("66b8ffff", |cb| mov(cb, AX, uimm_opnd(u16::MAX.into()))); - + let cb03 = compile(|cb| mov(cb, AX, uimm_opnd(1))); + let cb04 = compile(|cb| mov(cb, AX, uimm_opnd(u16::MAX.into()))); // MOV EAX, imm32 - check_bytes("b801000000", |cb| mov(cb, EAX, uimm_opnd(1))); - check_bytes("b8ffffffff", |cb| mov(cb, EAX, uimm_opnd(u32::MAX.into()))); - check_bytes("41b800000000", |cb| mov(cb, R8, uimm_opnd(0))); - check_bytes("41b8ffffffff", |cb| mov(cb, R8, uimm_opnd(0xFF_FF_FF_FF))); - + let cb05 = compile(|cb| mov(cb, EAX, uimm_opnd(1))); + let cb06 = compile(|cb| mov(cb, EAX, uimm_opnd(u32::MAX.into()))); + let cb07 = compile(|cb| mov(cb, R8, uimm_opnd(0))); + let cb08 = compile(|cb| mov(cb, R8, uimm_opnd(0xFF_FF_FF_FF))); // MOV RAX, imm64, will move down into EAX since it fits into 32 bits - check_bytes("b801000000", |cb| mov(cb, RAX, uimm_opnd(1))); - check_bytes("b8ffffffff", |cb| mov(cb, RAX, uimm_opnd(u32::MAX.into()))); - + let cb09 = compile(|cb| mov(cb, RAX, uimm_opnd(1))); + let cb10 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX.into()))); // MOV RAX, imm64, will not move down into EAX since it does not fit into 32 bits - check_bytes("48b80000000001000000", |cb| mov(cb, RAX, uimm_opnd(u32::MAX as u64 + 1))); - check_bytes("48b8ffffffffffffffff", |cb| mov(cb, RAX, uimm_opnd(u64::MAX))); - check_bytes("49b8ffffffffffffffff", |cb| mov(cb, R8, uimm_opnd(u64::MAX))); - + let cb11 = compile(|cb| mov(cb, RAX, uimm_opnd(u32::MAX as u64 + 1))); + let cb12 = compile(|cb| mov(cb, RAX, uimm_opnd(u64::MAX))); + let cb13 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); // MOV r8, imm8 - check_bytes("41b001", |cb| mov(cb, R8B, uimm_opnd(1))); - check_bytes("41b0ff", |cb| mov(cb, R8B, uimm_opnd(u8::MAX.into()))); - + let cb14 = compile(|cb| mov(cb, R8B, uimm_opnd(1))); + let cb15 = compile(|cb| mov(cb, R8B, uimm_opnd(u8::MAX.into()))); // MOV r16, imm16 - check_bytes("6641b80100", |cb| mov(cb, R8W, uimm_opnd(1))); - check_bytes("6641b8ffff", |cb| mov(cb, R8W, uimm_opnd(u16::MAX.into()))); - + let cb16 = compile(|cb| mov(cb, R8W, uimm_opnd(1))); + let cb17 = compile(|cb| mov(cb, R8W, uimm_opnd(u16::MAX.into()))); // MOV r32, imm32 - check_bytes("41b801000000", |cb| mov(cb, R8D, uimm_opnd(1))); - check_bytes("41b8ffffffff", |cb| mov(cb, R8D, uimm_opnd(u32::MAX.into()))); - + let cb18 = compile(|cb| mov(cb, R8D, uimm_opnd(1))); + let cb19 = compile(|cb| mov(cb, R8D, uimm_opnd(u32::MAX.into()))); // MOV r64, imm64, will move down into 32 bit since it fits into 32 bits - check_bytes("41b801000000", |cb| mov(cb, R8, uimm_opnd(1))); - + let cb20 = compile(|cb| mov(cb, R8, uimm_opnd(1))); // MOV r64, imm64, will not move down into 32 bit since it does not fit into 32 bits - check_bytes("49b8ffffffffffffffff", |cb| mov(cb, R8, uimm_opnd(u64::MAX))); + let cb21 = compile(|cb| mov(cb, R8, uimm_opnd(u64::MAX))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @" + 0x0: mov al, 1 + 0x0: mov al, 0xff + 0x0: mov ax, 1 + 0x0: mov ax, 0xffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: mov r8d, 0 + 0x0: mov r8d, 0xffffffff + 0x0: mov eax, 1 + 0x0: mov eax, 0xffffffff + 0x0: movabs rax, 0x100000000 + 0x0: mov rax, 0xffffffffffffffff + 0x0: mov r8, 0xffffffffffffffff + 0x0: mov r8b, 1 + 0x0: mov r8b, 0xff + 0x0: mov r8w, 1 + 0x0: mov r8w, 0xffff + 0x0: mov r8d, 1 + 0x0: mov r8d, 0xffffffff + 0x0: mov r8d, 1 + 0x0: mov r8, 0xffffffffffffffff + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19, cb20, cb21), @" + b001 + b0ff + 66b80100 + 66b8ffff + b801000000 + b8ffffffff + 41b800000000 + 41b8ffffffff + b801000000 + b8ffffffff + 48b80000000001000000 + 48c7c0ffffffff + 49c7c0ffffffff + 41b001 + 41b0ff + 6641b80100 + 6641b8ffff + 41b801000000 + 41b8ffffffff + 41b801000000 + 49c7c0ffffffff + "); } #[test] fn test_mov_iprel() { - check_bytes("8b0500000000", |cb| mov(cb, EAX, mem_opnd(32, RIP, 0))); - check_bytes("8b0505000000", |cb| mov(cb, EAX, mem_opnd(32, RIP, 5))); - - check_bytes("488b0500000000", |cb| mov(cb, RAX, mem_opnd(64, RIP, 0))); - check_bytes("488b0505000000", |cb| mov(cb, RAX, mem_opnd(64, RIP, 5))); - check_bytes("488b3d05000000", |cb| mov(cb, RDI, mem_opnd(64, RIP, 5))); + let cb1 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 0))); + let cb2 = compile(|cb| mov(cb, EAX, mem_opnd(32, RIP, 5))); + let cb3 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 0))); + let cb4 = compile(|cb| mov(cb, RAX, mem_opnd(64, RIP, 5))); + let cb5 = compile(|cb| mov(cb, RDI, mem_opnd(64, RIP, 5))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: mov eax, dword ptr [rip] + 0x0: mov eax, dword ptr [rip + 5] + 0x0: mov rax, qword ptr [rip] + 0x0: mov rax, qword ptr [rip + 5] + 0x0: mov rdi, qword ptr [rip + 5] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 8b0500000000 + 8b0505000000 + 488b0500000000 + 488b0505000000 + 488b3d05000000 + "); } #[test] fn test_movsx() { - check_bytes("660fbec0", |cb| movsx(cb, AX, AL)); - check_bytes("0fbed0", |cb| movsx(cb, EDX, AL)); - check_bytes("480fbec3", |cb| movsx(cb, RAX, BL)); - check_bytes("0fbfc8", |cb| movsx(cb, ECX, AX)); - check_bytes("4c0fbed9", |cb| movsx(cb, R11, CL)); - check_bytes("4c6354240c", |cb| movsx(cb, R10, mem_opnd(32, RSP, 12))); - check_bytes("480fbe0424", |cb| movsx(cb, RAX, mem_opnd(8, RSP, 0))); - check_bytes("490fbf5504", |cb| movsx(cb, RDX, mem_opnd(16, R13, 4))); + let cb1 = compile(|cb| movsx(cb, AX, AL)); + let cb2 = compile(|cb| movsx(cb, EDX, AL)); + let cb3 = compile(|cb| movsx(cb, RAX, BL)); + let cb4 = compile(|cb| movsx(cb, ECX, AX)); + let cb5 = compile(|cb| movsx(cb, R11, CL)); + let cb6 = compile(|cb| movsx(cb, R10, mem_opnd(32, RSP, 12))); + let cb7 = compile(|cb| movsx(cb, RAX, mem_opnd(8, RSP, 0))); + let cb8 = compile(|cb| movsx(cb, RDX, mem_opnd(16, R13, 4))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 0x0: movsx ax, al + 0x0: movsx edx, al + 0x0: movsx rax, bl + 0x0: movsx ecx, ax + 0x0: movsx r11, cl + 0x0: movsxd r10, dword ptr [rsp + 0xc] + 0x0: movsx rax, byte ptr [rsp] + 0x0: movsx rdx, word ptr [r13 + 4] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 660fbec0 + 0fbed0 + 480fbec3 + 0fbfc8 + 4c0fbed9 + 4c6354240c + 480fbe0424 + 490fbf5504 + "); } #[test] fn test_nop() { - check_bytes("90", |cb| nop(cb, 1)); - check_bytes("6690", |cb| nop(cb, 2)); - check_bytes("0f1f00", |cb| nop(cb, 3)); - check_bytes("0f1f4000", |cb| nop(cb, 4)); - check_bytes("0f1f440000", |cb| nop(cb, 5)); - check_bytes("660f1f440000", |cb| nop(cb, 6)); - check_bytes("0f1f8000000000", |cb| nop(cb, 7)); - check_bytes("0f1f840000000000", |cb| nop(cb, 8)); - check_bytes("660f1f840000000000", |cb| nop(cb, 9)); - check_bytes("660f1f84000000000090", |cb| nop(cb, 10)); - check_bytes("660f1f8400000000006690", |cb| nop(cb, 11)); - check_bytes("660f1f8400000000000f1f00", |cb| nop(cb, 12)); + let cb01 = compile(|cb| nop(cb, 1)); + let cb02 = compile(|cb| nop(cb, 2)); + let cb03 = compile(|cb| nop(cb, 3)); + let cb04 = compile(|cb| nop(cb, 4)); + let cb05 = compile(|cb| nop(cb, 5)); + let cb06 = compile(|cb| nop(cb, 6)); + let cb07 = compile(|cb| nop(cb, 7)); + let cb08 = compile(|cb| nop(cb, 8)); + let cb09 = compile(|cb| nop(cb, 9)); + let cb10 = compile(|cb| nop(cb, 10)); + let cb11 = compile(|cb| nop(cb, 11)); + let cb12 = compile(|cb| nop(cb, 12)); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @" + 0x0: nop + 0x0: nop + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop dword ptr [rax] + 0x0: nop dword ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop + 0x0: nop word ptr [rax + rax] + 0x9: nop dword ptr [rax] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12), @" + 90 + 6690 + 0f1f00 + 0f1f4000 + 0f1f440000 + 660f1f440000 + 0f1f8000000000 + 0f1f840000000000 + 660f1f840000000000 + 660f1f84000000000090 + 660f1f8400000000006690 + 660f1f8400000000000f1f00 + "); } #[test] fn test_not() { - check_bytes("66f7d0", |cb| not(cb, AX)); - check_bytes("f7d0", |cb| not(cb, EAX)); - check_bytes("49f71424", |cb| not(cb, mem_opnd(64, R12, 0))); - check_bytes("f794242d010000", |cb| not(cb, mem_opnd(32, RSP, 301))); - check_bytes("f71424", |cb| not(cb, mem_opnd(32, RSP, 0))); - check_bytes("f7542403", |cb| not(cb, mem_opnd(32, RSP, 3))); - check_bytes("f75500", |cb| not(cb, mem_opnd(32, RBP, 0))); - check_bytes("f7550d", |cb| not(cb, mem_opnd(32, RBP, 13))); - check_bytes("48f7d0", |cb| not(cb, RAX)); - check_bytes("49f7d3", |cb| not(cb, R11)); - check_bytes("f710", |cb| not(cb, mem_opnd(32, RAX, 0))); - check_bytes("f716", |cb| not(cb, mem_opnd(32, RSI, 0))); - check_bytes("f717", |cb| not(cb, mem_opnd(32, RDI, 0))); - check_bytes("f75237", |cb| not(cb, mem_opnd(32, RDX, 55))); - check_bytes("f79239050000", |cb| not(cb, mem_opnd(32, RDX, 1337))); - check_bytes("f752c9", |cb| not(cb, mem_opnd(32, RDX, -55))); - check_bytes("f792d5fdffff", |cb| not(cb, mem_opnd(32, RDX, -555))); + let cb01 = compile(|cb| not(cb, AX)); + let cb02 = compile(|cb| not(cb, EAX)); + let cb03 = compile(|cb| not(cb, mem_opnd(64, R12, 0))); + let cb04 = compile(|cb| not(cb, mem_opnd(32, RSP, 301))); + let cb05 = compile(|cb| not(cb, mem_opnd(32, RSP, 0))); + let cb06 = compile(|cb| not(cb, mem_opnd(32, RSP, 3))); + let cb07 = compile(|cb| not(cb, mem_opnd(32, RBP, 0))); + let cb08 = compile(|cb| not(cb, mem_opnd(32, RBP, 13))); + let cb09 = compile(|cb| not(cb, RAX)); + let cb10 = compile(|cb| not(cb, R11)); + let cb11 = compile(|cb| not(cb, mem_opnd(32, RAX, 0))); + let cb12 = compile(|cb| not(cb, mem_opnd(32, RSI, 0))); + let cb13 = compile(|cb| not(cb, mem_opnd(32, RDI, 0))); + let cb14 = compile(|cb| not(cb, mem_opnd(32, RDX, 55))); + let cb15 = compile(|cb| not(cb, mem_opnd(32, RDX, 1337))); + let cb16 = compile(|cb| not(cb, mem_opnd(32, RDX, -55))); + let cb17 = compile(|cb| not(cb, mem_opnd(32, RDX, -555))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @" + 0x0: not ax + 0x0: not eax + 0x0: not qword ptr [r12] + 0x0: not dword ptr [rsp + 0x12d] + 0x0: not dword ptr [rsp] + 0x0: not dword ptr [rsp + 3] + 0x0: not dword ptr [rbp] + 0x0: not dword ptr [rbp + 0xd] + 0x0: not rax + 0x0: not r11 + 0x0: not dword ptr [rax] + 0x0: not dword ptr [rsi] + 0x0: not dword ptr [rdi] + 0x0: not dword ptr [rdx + 0x37] + 0x0: not dword ptr [rdx + 0x539] + 0x0: not dword ptr [rdx - 0x37] + 0x0: not dword ptr [rdx - 0x22b] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17), @" + 66f7d0 + f7d0 + 49f71424 + f794242d010000 + f71424 + f7542403 + f75500 + f7550d + 48f7d0 + 49f7d3 + f710 + f716 + f717 + f75237 + f79239050000 + f752c9 + f792d5fdffff + "); } #[test] fn test_or() { - check_bytes("09f2", |cb| or(cb, EDX, ESI)); + let cb = compile(|cb| or(cb, EDX, ESI)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or edx, esi"); + assert_snapshot!(cb.hexdump(), @"09f2"); } #[test] fn test_pop() { - check_bytes("58", |cb| pop(cb, RAX)); - check_bytes("5b", |cb| pop(cb, RBX)); - check_bytes("5c", |cb| pop(cb, RSP)); - check_bytes("5d", |cb| pop(cb, RBP)); - check_bytes("415c", |cb| pop(cb, R12)); - check_bytes("8f00", |cb| pop(cb, mem_opnd(64, RAX, 0))); - check_bytes("418f00", |cb| pop(cb, mem_opnd(64, R8, 0))); - check_bytes("418f4003", |cb| pop(cb, mem_opnd(64, R8, 3))); - check_bytes("8f44c803", |cb| pop(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); - check_bytes("418f44c803", |cb| pop(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + let cb01 = compile(|cb| pop(cb, RAX)); + let cb02 = compile(|cb| pop(cb, RBX)); + let cb03 = compile(|cb| pop(cb, RSP)); + let cb04 = compile(|cb| pop(cb, RBP)); + let cb05 = compile(|cb| pop(cb, R12)); + let cb06 = compile(|cb| pop(cb, mem_opnd(64, RAX, 0))); + let cb07 = compile(|cb| pop(cb, mem_opnd(64, R8, 0))); + let cb08 = compile(|cb| pop(cb, mem_opnd(64, R8, 3))); + let cb09 = compile(|cb| pop(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb10 = compile(|cb| pop(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @" + 0x0: pop rax + 0x0: pop rbx + 0x0: pop rsp + 0x0: pop rbp + 0x0: pop r12 + 0x0: pop qword ptr [rax] + 0x0: pop qword ptr [r8] + 0x0: pop qword ptr [r8 + 3] + 0x0: pop qword ptr [rax + rcx*8 + 3] + 0x0: pop qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10), @" + 58 + 5b + 5c + 5d + 415c + 8f00 + 418f00 + 418f4003 + 8f44c803 + 418f44c803 + "); } #[test] fn test_push() { - check_bytes("50", |cb| push(cb, RAX)); - check_bytes("53", |cb| push(cb, RBX)); - check_bytes("4154", |cb| push(cb, R12)); - check_bytes("ff30", |cb| push(cb, mem_opnd(64, RAX, 0))); - check_bytes("41ff30", |cb| push(cb, mem_opnd(64, R8, 0))); - check_bytes("41ff7003", |cb| push(cb, mem_opnd(64, R8, 3))); - check_bytes("ff74c803", |cb| push(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); - check_bytes("41ff74c803", |cb| push(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + let cb1 = compile(|cb| push(cb, RAX)); + let cb2 = compile(|cb| push(cb, RBX)); + let cb3 = compile(|cb| push(cb, R12)); + let cb4 = compile(|cb| push(cb, mem_opnd(64, RAX, 0))); + let cb5 = compile(|cb| push(cb, mem_opnd(64, R8, 0))); + let cb6 = compile(|cb| push(cb, mem_opnd(64, R8, 3))); + let cb7 = compile(|cb| push(cb, mem_opnd_sib(64, RAX, RCX, 8, 3))); + let cb8 = compile(|cb| push(cb, mem_opnd_sib(64, R8, RCX, 8, 3))); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 0x0: push rax + 0x0: push rbx + 0x0: push r12 + 0x0: push qword ptr [rax] + 0x0: push qword ptr [r8] + 0x0: push qword ptr [r8 + 3] + 0x0: push qword ptr [rax + rcx*8 + 3] + 0x0: push qword ptr [r8 + rcx*8 + 3] + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5, cb6, cb7, cb8), @" + 50 + 53 + 4154 + ff30 + 41ff30 + 41ff7003 + ff74c803 + 41ff74c803 + "); } #[test] fn test_ret() { - check_bytes("c3", |cb| ret(cb)); + let cb = compile(ret); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ret"); + assert_snapshot!(cb.hexdump(), @"c3"); } #[test] fn test_sal() { - check_bytes("66d1e1", |cb| sal(cb, CX, uimm_opnd(1))); - check_bytes("d1e1", |cb| sal(cb, ECX, uimm_opnd(1))); - check_bytes("c1e505", |cb| sal(cb, EBP, uimm_opnd(5))); - check_bytes("d1642444", |cb| sal(cb, mem_opnd(32, RSP, 68), uimm_opnd(1))); - check_bytes("48d3e1", |cb| sal(cb, RCX, CL)); + let cb1 = compile(|cb| sal(cb, CX, uimm_opnd(1))); + let cb2 = compile(|cb| sal(cb, ECX, uimm_opnd(1))); + let cb3 = compile(|cb| sal(cb, EBP, uimm_opnd(5))); + let cb4 = compile(|cb| sal(cb, mem_opnd(32, RSP, 68), uimm_opnd(1))); + let cb5 = compile(|cb| sal(cb, RCX, CL)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4, cb5), @" + 0x0: shl cx, 1 + 0x0: shl ecx, 1 + 0x0: shl ebp, 5 + 0x0: shl dword ptr [rsp + 0x44], 1 + 0x0: shl rcx, cl + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4, cb5), @" + 66d1e1 + d1e1 + c1e505 + d1642444 + 48d3e1 + "); } #[test] fn test_sar() { - check_bytes("d1fa", |cb| sar(cb, EDX, uimm_opnd(1))); + let cb = compile(|cb| sar(cb, EDX, uimm_opnd(1))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sar edx, 1"); + assert_snapshot!(cb.hexdump(), @"d1fa"); } #[test] fn test_shr() { - check_bytes("49c1ee07", |cb| shr(cb, R14, uimm_opnd(7))); + let cb = compile(|cb| shr(cb, R14, uimm_opnd(7))); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: shr r14, 7"); + assert_snapshot!(cb.hexdump(), @"49c1ee07"); } #[test] fn test_sub() { - check_bytes("83e801", |cb| sub(cb, EAX, imm_opnd(1))); - check_bytes("4883e802", |cb| sub(cb, RAX, imm_opnd(2))); + let cb1 = compile(|cb| sub(cb, EAX, imm_opnd(1))); + let cb2 = compile(|cb| sub(cb, RAX, imm_opnd(2))); + + assert_disasm_snapshot!(disasms!(cb1, cb2), @" + 0x0: sub eax, 1 + 0x0: sub rax, 2 + "); + + assert_snapshot!(hexdumps!(cb1, cb2), @" + 83e801 + 4883e802 + "); } #[test] @@ -374,44 +842,103 @@ fn test_sub() { fn test_sub_uimm_too_large() { // This immediate becomes a different value after // sign extension, so not safe to encode. - check_bytes("ff", |cb| sub(cb, RCX, uimm_opnd(0x8000_0000))); + compile(|cb| sub(cb, RCX, uimm_opnd(0x8000_0000))); } #[test] fn test_test() { - check_bytes("84c0", |cb| test(cb, AL, AL)); - check_bytes("6685c0", |cb| test(cb, AX, AX)); - check_bytes("f6c108", |cb| test(cb, CL, uimm_opnd(8))); - check_bytes("f6c207", |cb| test(cb, DL, uimm_opnd(7))); - check_bytes("f6c108", |cb| test(cb, RCX, uimm_opnd(8))); - check_bytes("f6420808", |cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(8))); - check_bytes("f64208ff", |cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(255))); - check_bytes("66f7c2ffff", |cb| test(cb, DX, uimm_opnd(0xffff))); - check_bytes("66f74208ffff", |cb| test(cb, mem_opnd(16, RDX, 8), uimm_opnd(0xffff))); - check_bytes("f60601", |cb| test(cb, mem_opnd(8, RSI, 0), uimm_opnd(1))); - check_bytes("f6461001", |cb| test(cb, mem_opnd(8, RSI, 16), uimm_opnd(1))); - check_bytes("f646f001", |cb| test(cb, mem_opnd(8, RSI, -16), uimm_opnd(1))); - check_bytes("854640", |cb| test(cb, mem_opnd(32, RSI, 64), EAX)); - check_bytes("4885472a", |cb| test(cb, mem_opnd(64, RDI, 42), RAX)); - check_bytes("4885c0", |cb| test(cb, RAX, RAX)); - check_bytes("4885f0", |cb| test(cb, RAX, RSI)); - check_bytes("48f74640f7ffffff", |cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08))); - check_bytes("48f7464008000000", |cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(0x08))); - check_bytes("48f7c108000000", |cb| test(cb, RCX, imm_opnd(0x08))); - //check_bytes("48a9f7ffff0f", |cb| test(cb, RAX, imm_opnd(0x0FFFFFF7))); + let cb01 = compile(|cb| test(cb, AL, AL)); + let cb02 = compile(|cb| test(cb, AX, AX)); + let cb03 = compile(|cb| test(cb, CL, uimm_opnd(8))); + let cb04 = compile(|cb| test(cb, DL, uimm_opnd(7))); + let cb05 = compile(|cb| test(cb, RCX, uimm_opnd(8))); + let cb06 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(8))); + let cb07 = compile(|cb| test(cb, mem_opnd(8, RDX, 8), uimm_opnd(255))); + let cb08 = compile(|cb| test(cb, DX, uimm_opnd(0xffff))); + let cb09 = compile(|cb| test(cb, mem_opnd(16, RDX, 8), uimm_opnd(0xffff))); + let cb10 = compile(|cb| test(cb, mem_opnd(8, RSI, 0), uimm_opnd(1))); + let cb11 = compile(|cb| test(cb, mem_opnd(8, RSI, 16), uimm_opnd(1))); + let cb12 = compile(|cb| test(cb, mem_opnd(8, RSI, -16), uimm_opnd(1))); + let cb13 = compile(|cb| test(cb, mem_opnd(32, RSI, 64), EAX)); + let cb14 = compile(|cb| test(cb, mem_opnd(64, RDI, 42), RAX)); + let cb15 = compile(|cb| test(cb, RAX, RAX)); + let cb16 = compile(|cb| test(cb, RAX, RSI)); + let cb17 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08))); + let cb18 = compile(|cb| test(cb, mem_opnd(64, RSI, 64), imm_opnd(0x08))); + let cb19 = compile(|cb| test(cb, RCX, imm_opnd(0x08))); + + assert_disasm_snapshot!(disasms!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @" + 0x0: test al, al + 0x0: test ax, ax + 0x0: test cl, 8 + 0x0: test dl, 7 + 0x0: test cl, 8 + 0x0: test byte ptr [rdx + 8], 8 + 0x0: test byte ptr [rdx + 8], 0xff + 0x0: test dx, 0xffff + 0x0: test word ptr [rdx + 8], 0xffff + 0x0: test byte ptr [rsi], 1 + 0x0: test byte ptr [rsi + 0x10], 1 + 0x0: test byte ptr [rsi - 0x10], 1 + 0x0: test dword ptr [rsi + 0x40], eax + 0x0: test qword ptr [rdi + 0x2a], rax + 0x0: test rax, rax + 0x0: test rax, rsi + 0x0: test qword ptr [rsi + 0x40], -9 + 0x0: test qword ptr [rsi + 0x40], 8 + 0x0: test rcx, 8 + "); + + assert_snapshot!(hexdumps!(cb01, cb02, cb03, cb04, cb05, cb06, cb07, cb08, cb09, cb10, cb11, cb12, cb13, cb14, cb15, cb16, cb17, cb18, cb19), @" + 84c0 + 6685c0 + f6c108 + f6c207 + f6c108 + f6420808 + f64208ff + 66f7c2ffff + 66f74208ffff + f60601 + f6461001 + f646f001 + 854640 + 4885472a + 4885c0 + 4885f0 + 48f74640f7ffffff + 48f7464008000000 + 48f7c108000000 + "); } #[test] fn test_xchg() { - check_bytes("4891", |cb| xchg(cb, RAX, RCX)); - check_bytes("4995", |cb| xchg(cb, RAX, R13)); - check_bytes("4887d9", |cb| xchg(cb, RCX, RBX)); - check_bytes("4d87f9", |cb| xchg(cb, R9, R15)); + let cb1 = compile(|cb| xchg(cb, RAX, RCX)); + let cb2 = compile(|cb| xchg(cb, RAX, R13)); + let cb3 = compile(|cb| xchg(cb, RCX, RBX)); + let cb4 = compile(|cb| xchg(cb, R9, R15)); + + assert_disasm_snapshot!(disasms!(cb1, cb2, cb3, cb4), @" + 0x0: xchg rcx, rax + 0x0: xchg r13, rax + 0x0: xchg rcx, rbx + 0x0: xchg r9, r15 + "); + + assert_snapshot!(hexdumps!(cb1, cb2, cb3, cb4), @" + 4891 + 4995 + 4887d9 + 4d87f9 + "); } #[test] fn test_xor() { - check_bytes("31c0", |cb| xor(cb, EAX, EAX)); + let cb = compile(|cb| xor(cb, EAX, EAX)); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor eax, eax"); + assert_snapshot!(cb.hexdump(), @"31c0"); } #[test] @@ -437,25 +964,3 @@ fn basic_capstone_usage() -> std::result::Result<(), capstone::Error> { )), } } - -#[test] -#[ignore] -#[cfg(feature = "disasm")] -fn block_comments() { - let mut cb = super::CodeBlock::new_dummy(); - - let first_write_ptr = cb.get_write_ptr().raw_addr(&cb); - cb.add_comment("Beginning"); - xor(&mut cb, EAX, EAX); // 2 bytes long - let second_write_ptr = cb.get_write_ptr().raw_addr(&cb); - cb.add_comment("Two bytes in"); - cb.add_comment("Still two bytes in"); - cb.add_comment("Still two bytes in"); // Duplicate, should be ignored - test(&mut cb, mem_opnd(64, RSI, 64), imm_opnd(!0x08)); // 8 bytes long - let third_write_ptr = cb.get_write_ptr().raw_addr(&cb); - cb.add_comment("Ten bytes in"); - - assert_eq!(&vec!( "Beginning".to_string() ), cb.comments_at(first_write_ptr).unwrap()); - assert_eq!(&vec!( "Two bytes in".to_string(), "Still two bytes in".to_string() ), cb.comments_at(second_write_ptr).unwrap()); - assert_eq!(&vec!( "Ten bytes in".to_string() ), cb.comments_at(third_write_ptr).unwrap()); -} diff --git a/zjit/src/assertions.rs b/zjit/src/assertions.rs deleted file mode 100644 index 0dacc938fc..0000000000 --- a/zjit/src/assertions.rs +++ /dev/null @@ -1,21 +0,0 @@ -/// Assert that CodeBlock has the code specified with hex. In addition, if tested with -/// `cargo test --all-features`, it also checks it generates the specified disasm. -#[cfg(test)] -macro_rules! assert_disasm { - ($cb:expr, $hex:expr, $disasm:expr) => { - #[cfg(feature = "disasm")] - { - use $crate::disasm::disasm_addr_range; - use $crate::cruby::unindent; - let disasm = disasm_addr_range( - &$cb, - $cb.get_ptr(0).raw_addr(&$cb), - $cb.get_write_ptr().raw_addr(&$cb), - ); - assert_eq!(unindent(&disasm, false), unindent(&$disasm, true)); - } - assert_eq!(format!("{:x}", $cb), $hex); - }; -} -#[cfg(test)] -pub(crate) use assert_disasm; diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index c60ec53285..32ba1e3de0 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1,15 +1,21 @@ -use std::mem::take; - use crate::asm::{CodeBlock, Label}; use crate::asm::arm64::*; +use crate::codegen::split_patch_point; use crate::cruby::*; use crate::backend::lir::*; +use crate::options::asm_dump; +use crate::stats::{CompileError, trace_compile_phase}; use crate::virtualmem::CodePtr; use crate::cast::*; // Use the arm64 register type for this platform pub type Reg = A64Reg; +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_no } +} + // Callee-saved registers pub const CFP: Opnd = Opnd::Reg(X19_REG); pub const EC: Opnd = Opnd::Reg(X20_REG); @@ -25,6 +31,10 @@ pub const C_ARG_OPNDS: [Opnd; 6] = [ Opnd::Reg(X5_REG) ]; +// Make sure we're using the same c args everywhere +const _: () = ::core::assert!(C_ARG_OPNDS.len() == C_ARG_REGS.len()); +const _: () = ::core::assert!(C_ARG_OPNDS.len() == C_ARG_REGREGS.len()); + // C return value register on this platform pub const C_RET_REG: Reg = X0_REG; pub const C_RET_OPND: Opnd = Opnd::Reg(X0_REG); @@ -72,12 +82,14 @@ impl From<Opnd> for A64Opnd { Opnd::Mem(Mem { base: MemBase::VReg(_), .. }) => { panic!("attempted to lower an Opnd::Mem with a MemBase::VReg base") }, + Opnd::Mem(Mem { base: MemBase::Stack { .. } | MemBase::StackIndirect { .. }, .. }) => { + panic!("attempted to lower an Opnd::Mem with a MemBase::Stack/StackIndirect base") + }, Opnd::VReg { .. } => panic!("attempted to lower an Opnd::VReg"), Opnd::Value(_) => panic!("attempted to lower an Opnd::Value"), Opnd::None => panic!( "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." ), - } } } @@ -98,7 +110,7 @@ fn emit_jmp_ptr_with_invalidation(cb: &mut CodeBlock, dst_ptr: CodePtr) { let start = cb.get_write_ptr(); emit_jmp_ptr(cb, dst_ptr, true); let end = cb.get_write_ptr(); - unsafe { rb_zjit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; + unsafe { rb_jit_icache_invalidate(start.raw_ptr(cb) as _, end.raw_ptr(cb) as _) }; } fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { @@ -113,8 +125,8 @@ fn emit_jmp_ptr(cb: &mut CodeBlock, dst_ptr: CodePtr, padding: bool) { b(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); 1 } else { - let num_insns = emit_load_value(cb, Assembler::SCRATCH0, dst_addr as u64); - br(cb, Assembler::SCRATCH0); + let num_insns = emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); num_insns + 1 }; @@ -139,17 +151,17 @@ fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { // If the value fits into a single movz // instruction, then we'll use that. movz(cb, rd, A64Opnd::new_uimm(current), 0); - return 1; + 1 } else if u16::try_from(!value).is_ok() { // For small negative values, use a single movn movn(cb, rd, A64Opnd::new_uimm(!value), 0); - return 1; + 1 } else if BitmaskImmediate::try_from(current).is_ok() { // Otherwise, if the immediate can be encoded // with the special bitmask immediate encoding, // we'll use that. mov(cb, rd, A64Opnd::new_uimm(current)); - return 1; + 1 } else { // Finally we'll fall back to encoding the value // using movz for the first 16 bits and movk for @@ -175,14 +187,14 @@ fn emit_load_value(cb: &mut CodeBlock, rd: A64Opnd, value: u64) -> usize { movk(cb, rd, A64Opnd::new_uimm(current & 0xffff), 48); num_insns += 1; } - return num_insns; + num_insns } } /// List of registers that can be used for register allocation. /// This has the same number of registers for x86_64 and arm64. -/// SCRATCH0 and SCRATCH1 are excluded. -pub const ALLOC_REGS: &'static [Reg] = &[ +/// SCRATCH_OPND, SCRATCH1_OPND, and EMIT_OPND are excluded. +pub const ALLOC_REGS: &[Reg] = &[ X0_REG, X1_REG, X2_REG, @@ -193,17 +205,32 @@ pub const ALLOC_REGS: &'static [Reg] = &[ X12_REG, ]; -impl Assembler -{ - /// Special scratch registers for intermediate processing. - /// This register is call-clobbered (so we don't have to save it before using it). - /// Avoid using if you can since this is used to lower [Insn] internally and - /// so conflicts are possible. - pub const SCRATCH_REG: Reg = X16_REG; - const SCRATCH0_REG: Reg = Self::SCRATCH_REG; - const SCRATCH1_REG: Reg = X17_REG; - const SCRATCH0: A64Opnd = A64Opnd::Reg(Self::SCRATCH0_REG); - const SCRATCH1: A64Opnd = A64Opnd::Reg(Self::SCRATCH1_REG); +/// Special scratch registers for intermediate processing. They should be used only by +/// [`Assembler::arm64_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(X15_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(X17_REG); + +/// A scratch register available for use by resolve_ssa to break register copy cycles. +/// Must not overlap with ALLOC_REGS or other preserved registers. +pub const SCRATCH_REG: Reg = X15_REG; +const SCRATCH2_OPND: Opnd = Opnd::Reg(X14_REG); + +impl Assembler { + const MAX_FRAME_STACK_SLOTS: usize = 2048; + + /// Special register for intermediate processing in arm64_emit. It should be used only by arm64_emit. + const EMIT_REG: Reg = X16_REG; + const EMIT_OPND: A64Opnd = A64Opnd::Reg(Self::EMIT_REG); + + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } /// Get the list of registers from which we will allocate on this platform pub fn get_alloc_regs() -> Vec<Reg> { @@ -237,8 +264,8 @@ impl Assembler if mem_disp_fits_bits(mem.disp) { opnd } else { - let base = asm.lea(opnd); - Opnd::mem(64, base, 0) + let base = asm.lea(Opnd::Mem(Mem { num_bits: 64, ..mem })); + Opnd::mem(mem.num_bits, base, 0) } }, _ => unreachable!("Can only split memory addresses.") @@ -370,38 +397,34 @@ impl Assembler } } - let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); - let mut iterator = self.insns.into_iter().enumerate().peekable(); - let mut asm_local = Assembler::new_with_label_names(take(&mut self.label_names), live_ranges.len()); + let mut asm_local = Assembler::new_with_asm(&self); + let mut iterator = self.instruction_iterator(); let asm = &mut asm_local; - while let Some((index, mut insn)) = iterator.next() { + while let Some((_index, mut insn)) = iterator.next(asm) { // Here we're going to map the operands of the instruction to load // any Opnd::Value operands into registers if they are heap objects // such that only the Op::Load instruction needs to handle that // case. If the values aren't heap objects then we'll treat them as // if they were just unsigned integer. let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. }); - let mut opnd_iter = insn.opnd_iter_mut(); - - while let Some(opnd) = opnd_iter.next() { - match opnd { - Opnd::Value(value) => { - if value.special_const_p() { - *opnd = Opnd::UImm(value.as_u64()); - } else if !is_load { - *opnd = asm.load(*opnd); - } - }, - _ => {} + let is_jump = insn.is_jump(); + + insn.for_each_operand_mut(|opnd| { + if let Opnd::Value(value) = opnd { + if value.special_const_p() { + *opnd = Opnd::UImm(value.as_u64()); + } else if !is_load && !is_jump { + *opnd = asm.load(*opnd); + } }; - } + }); // We are replacing instructions here so we know they are already // being used. It is okay not to use their output here. #[allow(unused_must_use)] match &mut insn { - Insn::Add { left, right, out } => { + Insn::Add { left, right, .. } => { match (*left, *right) { // When one operand is a register, legalize the other operand // into possibly an immdiate and swap the order if necessary. @@ -412,34 +435,29 @@ impl Assembler *right = split_shifted_immediate(asm, other_opnd); // Now `right` is either a register or an immediate, both can try to // merge with a subsequent mov. - merge_three_reg_mov(&live_ranges, &mut iterator, left, left, out); + asm.push_insn(insn); } _ => { *left = split_load_operand(asm, *left); *right = split_shifted_immediate(asm, *right); - merge_three_reg_mov(&live_ranges, &mut iterator, left, right, out); + asm.push_insn(insn); } } } - Insn::Sub { left, right, out } => { + Insn::Sub { left, right, .. } => { *left = split_load_operand(asm, *left); *right = split_shifted_immediate(asm, *right); - // Now `right` is either a register or an immediate, - // both can try to merge with a subsequent mov. - merge_three_reg_mov(&live_ranges, &mut iterator, left, left, out); asm.push_insn(insn); } - Insn::And { left, right, out } | - Insn::Or { left, right, out } | - Insn::Xor { left, right, out } => { + Insn::And { left, right, .. } | + Insn::Or { left, right, .. } | + Insn::Xor { left, right, .. } => { let (opnd0, opnd1) = split_boolean_operands(asm, *left, *right); *left = opnd0; *right = opnd1; - merge_three_reg_mov(&live_ranges, &mut iterator, left, right, out); - asm.push_insn(insn); } /* @@ -471,34 +489,6 @@ impl Assembler iterator.next_unmapped(); // Pop merged jump instruction } */ - Insn::CCall { opnds, .. } => { - assert!(opnds.len() <= C_ARG_OPNDS.len()); - - // Load each operand into the corresponding argument - // register. - // Note: the iteration order is reversed to avoid corrupting x0, - // which is both the return value and first argument register - if !opnds.is_empty() { - let mut args: Vec<(Reg, Opnd)> = vec![]; - for (idx, opnd) in opnds.into_iter().enumerate().rev() { - // If the value that we're sending is 0, then we can use - // the zero register, so in this case we'll just send - // a UImm of 0 along as the argument to the move. - let value = match opnd { - Opnd::UImm(0) | Opnd::Imm(0) => Opnd::UImm(0), - Opnd::Mem(_) => split_memory_address(asm, *opnd), - _ => *opnd - }; - args.push((C_ARG_OPNDS[idx].unwrap_reg(), value)); - } - asm.parallel_mov(args); - } - - // Now we push the CCall without any arguments so that it - // just performs the call. - *opnds = vec![]; - asm.push_insn(insn); - }, Insn::Cmp { left, right } => { let opnd0 = split_load_operand(asm, *left); let opnd0 = split_less_than_32_cmp(asm, opnd0); @@ -534,37 +524,18 @@ impl Assembler } asm.cret(C_RET_OPND); }, - Insn::CSelZ { truthy, falsy, out } | - Insn::CSelNZ { truthy, falsy, out } | - Insn::CSelE { truthy, falsy, out } | - Insn::CSelNE { truthy, falsy, out } | - Insn::CSelL { truthy, falsy, out } | - Insn::CSelLE { truthy, falsy, out } | - Insn::CSelG { truthy, falsy, out } | - Insn::CSelGE { truthy, falsy, out } => { + Insn::CSelZ { truthy, falsy, .. } | + Insn::CSelNZ { truthy, falsy, .. } | + Insn::CSelE { truthy, falsy, .. } | + Insn::CSelNE { truthy, falsy, .. } | + Insn::CSelL { truthy, falsy, .. } | + Insn::CSelLE { truthy, falsy, .. } | + Insn::CSelG { truthy, falsy, .. } | + Insn::CSelGE { truthy, falsy, .. } => { let (opnd0, opnd1) = split_csel_operands(asm, *truthy, *falsy); *truthy = opnd0; *falsy = opnd1; - // Merge `csel` and `mov` into a single `csel` when possible - match iterator.peek().map(|(_, insn)| insn) { - Some(Insn::Mov { dest: Opnd::Reg(reg), src }) - if matches!(out, Opnd::VReg { .. }) && *out == *src && live_ranges[out.vreg_idx()].end() == index + 1 => { - *out = Opnd::Reg(*reg); - asm.push_insn(insn); - iterator.next(); // Pop merged Insn::Mov - } - _ => { - asm.push_insn(insn); - } - } - }, - Insn::IncrCounter { mem, value } => { - let counter_addr = match mem { - Opnd::Mem(_) => asm.lea(*mem), - _ => *mem - }; - - asm.incr_counter(counter_addr, *value); + asm.push_insn(insn); }, Insn::JmpOpnd(opnd) => { if let Opnd::Mem(_) = opnd { @@ -682,9 +653,255 @@ impl Assembler asm_local } + /// Split instructions using scratch registers. To maximize the use of the register pool for + /// VRegs, most splits should happen in [`Self::arm64_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so this + /// splits them and uses scratch registers for it. + /// Linearizes all blocks into a single giant block. + fn arm64_scratch_split(self) -> Assembler { + /// If opnd is Opnd::Mem with a too large disp, make the disp smaller using lea. + fn split_large_disp(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { + match opnd { + Opnd::Mem(Mem { num_bits, disp, .. }) if !mem_disp_fits_bits(disp) => { + asm.lea_into(scratch_opnd, opnd); + Opnd::mem(num_bits, scratch_opnd, 0) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem with MemBase::Stack, lower it to Opnd::Mem with MemBase::Reg, and split a large disp. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + let opnd = split_only_stack_membase(asm, opnd, scratch_opnd, stack_state); + split_large_disp(asm, opnd, scratch_opnd) + } + + /// split_stack_membase but without split_large_disp. This should be used only by lea, + /// whose lowering already handles large displacements in arm64_emit. + fn split_only_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + match opnd { + Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // Convert MemBase::Stack to MemBase::Reg(NATIVE_BASE_PTR) with the + // correct stack displacement. The stack slot value lives directly at + // [NATIVE_BASE_PTR + stack_disp], so we just adjust the base and + // combine displacements -- no indirection needed. Large + // displacements are handled by split_stack_membase(). + let Mem { base, disp: stack_disp, .. } = stack_state.stack_membase_to_mem(stack_membase); + Opnd::Mem(Mem { base, disp: stack_disp + opnd_disp, num_bits: opnd_num_bits }) + } + Opnd::Mem(Mem { base: MemBase::StackIndirect { stack_idx }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // The spilled value (a pointer) lives at a stack slot. Load it + // into a scratch register, then use the register as the base. + let stack_mem = stack_state.stack_membase_to_mem(MemBase::Stack { stack_idx, num_bits: 64 }); + let stack_opnd = split_large_disp(asm, Opnd::Mem(stack_mem), scratch_opnd); + asm.load_into(scratch_opnd, stack_opnd); + Opnd::Mem(Mem { + base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), + disp: opnd_disp, + num_bits: opnd_num_bits, + }) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem, lower it to scratch_opnd. You should use this when `opnd` is read by the instruction, not written. + fn split_memory_read(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + if let Opnd::Mem(_) = opnd { + let opnd = split_stack_membase(asm, opnd, scratch_opnd, stack_state); + let scratch_opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + asm.load_into(scratch_opnd, opnd); + scratch_opnd + } else { + opnd + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.num_vregs = self.num_vregs; + + // Create one giant block to linearize everything into + asm_local.new_block_without_id("linearized"); + + let asm = &mut asm_local; + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // Process each linearized instruction + for (idx, insn) in linearized_insns.iter().enumerate() { + let mut insn = insn.clone(); + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } | + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND, &stack_state); + *right = split_memory_read(asm, *right, SCRATCH1_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Mul { left, right, out } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND, &stack_state); + *right = split_memory_read(asm, *right, SCRATCH1_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + let reg_out = out.clone(); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + }; + + // If the next instruction is JoMul + if idx + 1 < linearized_insns.len() && matches!(linearized_insns[idx + 1], Insn::JoMul(_)) { + // Produce a register that is all zeros or all ones + // Based on the sign bit of the 64-bit mul result + asm.push_insn(Insn::RShift { out: SCRATCH0_OPND, opnd: reg_out, shift: Opnd::UImm(63) }); + } + } + Insn::LShift { opnd, out, .. } | + Insn::RShift { opnd, out, .. } => { + *opnd = split_memory_read(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Cmp { left, right } | + Insn::Test { left, right } => { + *left = split_memory_read(asm, *left, SCRATCH0_OPND, &stack_state); + *right = split_memory_read(asm, *right, SCRATCH1_OPND, &stack_state); + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + // For compile_exits, support splitting simple return values here + Insn::CRet(opnd) => { + match opnd { + Opnd::Reg(C_RET_REG) => {}, + _ => asm.load_into(C_RET_OPND, *opnd), + } + asm.cret(C_RET_OPND); + } + Insn::Lea { opnd, out } => { + *opnd = split_only_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + + asm.push_insn(insn); + + if let Some(mem_out) = mem_out { + let mem_out = split_stack_membase(asm, mem_out, SCRATCH1_OPND, &stack_state); + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Load { opnd, out } | + Insn::LoadInto { opnd, dest: out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + *out = split_stack_membase(asm, *out, SCRATCH1_OPND, &stack_state); + + if let Opnd::Mem(_) = out { + // If NATIVE_STACK_PTR is used as a source for Store, it's handled as xzr, storeing zero. + // To save the content of NATIVE_STACK_PTR, we need to load it into another register first. + if *opnd == NATIVE_STACK_PTR { + asm.load_into(SCRATCH0_OPND, NATIVE_STACK_PTR); + *opnd = SCRATCH0_OPND; + } + asm.store(*out, *opnd); + } else { + // If in and out are the same, this is a redundant mov + if opnd != out { + asm.push_insn(insn); + } + } + } + &mut Insn::IncrCounter { mem, value } => { + // Convert Opnd::const_ptr into Opnd::Mem. + // It's split here to support IncrCounter in compile_exits. + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.lea_into(SCRATCH0_OPND, Opnd::mem(64, SCRATCH0_OPND, 0)); + + // Create a local loop to atomically increment a counter using SCRATCH1_OPND to check if it succeeded. + // Note that arm64_emit will peek at the next Cmp to set a status into SCRATCH1_OPND on IncrCounter. + let label = asm.new_label("incr_counter_loop"); + asm.write_label(label.clone()); + asm.incr_counter(SCRATCH0_OPND, value); + asm.cmp(SCRATCH1_OPND, 0.into()); + asm.push_insn(Insn::Jne(label)); + } + Insn::Store { dest, src } => { + *dest = split_stack_membase(asm, *dest, SCRATCH0_OPND, &stack_state); + *src = split_stack_membase(asm, *src, SCRATCH1_OPND, &stack_state); + asm.push_insn(insn); + } + Insn::Mov { dest, src } => { + *src = split_stack_membase(asm, *src, SCRATCH0_OPND, &stack_state); + *dest = split_stack_membase(asm, *dest, SCRATCH1_OPND, &stack_state); + match dest { + Opnd::Reg(_) => asm.load_into(*dest, *src), + Opnd::Mem(_) => asm.store(*dest, *src), + _ => asm.push_insn(insn), + } + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + /// Emit platform-specific machine code /// Returns a list of GC offsets. Can return failure to signal caller to retry. - fn arm64_emit(&mut self, cb: &mut CodeBlock) -> Option<Vec<CodePtr>> { + fn arm64_emit(&mut self, cb: &mut CodeBlock) -> Result<Vec<CodePtr>, CompileError> { /// Determine how many instructions it will take to represent moving /// this value into a register. Note that the return value of this /// function must correspond to how many instructions are used to @@ -707,84 +924,87 @@ impl Assembler /// Emit a conditional jump instruction to a specific target. This is /// called when lowering any of the conditional jump instructions. - fn emit_conditional_jump<const CONDITION: u8>(cb: &mut CodeBlock, target: Target) { - match target { + fn emit_conditional_jump<const CONDITION: u8>(asm: &Assembler, cb: &mut CodeBlock, target: Target) { + fn generate_branch<const CONDITION: u8>(cb: &mut CodeBlock, src_addr: i64, dst_addr: i64) { + let num_insns = if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { + // If the jump offset fits into the conditional jump as + // an immediate value and it's properly aligned, then we + // can use the b.cond instruction directly. We're safe + // to use as i32 here since we already checked that it + // fits. + let bytes = (dst_addr - src_addr) as i32; + bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); + + // Here we're going to return 1 because we've only + // written out 1 instruction. + 1 + } else if b_offset_fits_bits((dst_addr - (src_addr + 4)) / 4) { // + 4 for bcond + // If the jump offset fits into the unconditional jump as + // an immediate value, we can use inverse b.cond + b. + // + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instruction used for branching. + bcond(cb, Condition::inverse(CONDITION), 2.into()); + b(cb, InstructionOffset::from_bytes((dst_addr - (src_addr + 4)) as i32)); // + 4 for bcond + + // We've only written out 2 instructions. + 2 + } else { + // Otherwise, we need to load the address into a + // register and use the branch register instruction. + let load_insns: i32 = emit_load_size(dst_addr as u64).into(); + + // We're going to write out the inverse condition so + // that if it doesn't match it will skip over the + // instructions used for branching. + bcond(cb, Condition::inverse(CONDITION), (load_insns + 2).into()); + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr as u64); + br(cb, Assembler::EMIT_OPND); + + // Here we'll return the number of instructions that it + // took to write out the destination address + 1 for the + // b.cond and 1 for the br. + load_insns + 2 + }; + + // We need to make sure we have at least 6 instructions for + // every kind of jump for invalidation purposes, so we're + // going to write out padding nop instructions here. + assert!(num_insns <= cb.conditional_jump_insns()); + (num_insns..cb.conditional_jump_insns()).for_each(|_| nop(cb)); + } + + let label = match target { Target::CodePtr(dst_ptr) => { let dst_addr = dst_ptr.as_offset(); let src_addr = cb.get_write_ptr().as_offset(); - - let num_insns = if bcond_offset_fits_bits((dst_addr - src_addr) / 4) { - // If the jump offset fits into the conditional jump as - // an immediate value and it's properly aligned, then we - // can use the b.cond instruction directly. We're safe - // to use as i32 here since we already checked that it - // fits. - let bytes = (dst_addr - src_addr) as i32; - bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); - - // Here we're going to return 1 because we've only - // written out 1 instruction. - 1 - } else if b_offset_fits_bits((dst_addr - (src_addr + 4)) / 4) { // + 4 for bcond - // If the jump offset fits into the unconditional jump as - // an immediate value, we can use inverse b.cond + b. - // - // We're going to write out the inverse condition so - // that if it doesn't match it will skip over the - // instruction used for branching. - bcond(cb, Condition::inverse(CONDITION), 2.into()); - b(cb, InstructionOffset::from_bytes((dst_addr - (src_addr + 4)) as i32)); // + 4 for bcond - - // We've only written out 2 instructions. - 2 - } else { - // Otherwise, we need to load the address into a - // register and use the branch register instruction. - let dst_addr = (dst_ptr.raw_ptr(cb) as usize).as_u64(); - let load_insns: i32 = emit_load_size(dst_addr).into(); - - // We're going to write out the inverse condition so - // that if it doesn't match it will skip over the - // instructions used for branching. - bcond(cb, Condition::inverse(CONDITION), (load_insns + 2).into()); - emit_load_value(cb, Assembler::SCRATCH0, dst_addr); - br(cb, Assembler::SCRATCH0); - - // Here we'll return the number of instructions that it - // took to write out the destination address + 1 for the - // b.cond and 1 for the br. - load_insns + 2 - }; - - if let Target::CodePtr(_) = target { - // We need to make sure we have at least 6 instructions for - // every kind of jump for invalidation purposes, so we're - // going to write out padding nop instructions here. - assert!(num_insns <= cb.conditional_jump_insns()); - for _ in num_insns..cb.conditional_jump_insns() { nop(cb); } - } - }, - Target::Label(label_idx) => { - // Here we're going to save enough space for ourselves and - // then come back and write the instruction once we know the - // offset. We're going to assume we can fit into a single - // b.cond instruction. It will panic otherwise. - cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { - let bytes: i32 = (dst_addr - (src_addr - 4)).try_into().unwrap(); - bcond(cb, CONDITION, InstructionOffset::from_bytes(bytes)); - }); + generate_branch::<CONDITION>(cb, src_addr, dst_addr); + return; }, + Target::Label(l) => l, + Target::Block(ref edge) => asm.block_label(edge.target), Target::SideExit { .. } => { - unreachable!("Target::SideExit should have been compiled by compile_side_exits") + unreachable!("Target::SideExit should have been compiled by compile_exits") }, }; + // Try to use a single B.cond instruction + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if bcond_offset_fits_bits(offset) { + bcond(cb, CONDITION, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); } /// Emit a CBZ or CBNZ which branches when a register is zero or non-zero - fn emit_cmp_zero_jump(_cb: &mut CodeBlock, _reg: A64Opnd, _branch_if_zero: bool, target: Target) { - if let Target::Label(_) = target { - unimplemented!("this should be re-implemented with Label for side exits"); - /* + fn emit_cmp_zero_jump(cb: &mut CodeBlock, reg: A64Opnd, branch_if_zero: bool, target: Target) { + if let Target::CodePtr(dst_ptr) = target { let dst_addr = dst_ptr.as_offset(); let src_addr = cb.get_write_ptr().as_offset(); @@ -812,11 +1032,9 @@ impl Assembler } else { cbz(cb, reg, InstructionOffset::from_insns(load_insns + 2)); } - emit_load_value(cb, Assembler::SCRATCH0, dst_addr); - br(cb, Assembler::SCRATCH0); - + emit_load_value(cb, Assembler::EMIT_OPND, dst_addr); + br(cb, Assembler::EMIT_OPND); } - */ } else { unreachable!("We should only generate Joz/Jonz with side-exit targets"); } @@ -825,7 +1043,9 @@ impl Assembler /// Do the address calculation of `out_reg = base_reg + disp` fn load_effective_address(cb: &mut CodeBlock, out: A64Opnd, base_reg_no: u8, disp: i32) { let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); - assert_ne!(31, out.unwrap_reg().reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + assert_ne!(base_reg_no, out_reg_no, "large displacement need a scratch register"); if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { // Use ADD/SUB if the displacement fits @@ -858,14 +1078,13 @@ impl Assembler gc_offsets.push(ptr_offset); } - /// Emit a push instruction for the given operand by adding to the stack - /// pointer and then storing the given value. + /// Push a value to the stack by subtracting from the stack pointer then storing, + /// leaving an 8-byte gap for alignment. fn emit_push(cb: &mut CodeBlock, opnd: A64Opnd) { str_pre(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); } - /// Emit a pop instruction into the given operand by loading the value - /// and then subtracting from the stack pointer. + /// Pop a value from the stack by loading `[sp]` then adding to the stack pointer. fn emit_pop(cb: &mut CodeBlock, opnd: A64Opnd) { ldr_post(cb, opnd, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); } @@ -880,8 +1099,13 @@ impl Assembler let mut last_patch_pos: Option<usize> = None; // For each instruction + // NOTE: At this point, the assembler should have been linearized into a single giant block + // by either resolve_parallel_mov_pass() or arm64_scratch_split(). let mut insn_idx: usize = 0; - while let Some(insn) = self.insns.get(insn_idx) { + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { match insn { Insn::Comment(text) => { cb.add_comment(text); @@ -934,7 +1158,9 @@ impl Assembler if slot_count > 0 { let slot_offset = (slot_count * SIZEOF_VALUE) as u64; // Bail when asked to reserve too many slots in one instruction. - ShiftedImmediate::try_from(slot_offset).ok()?; + if ShiftedImmediate::try_from(slot_offset).is_err() { + return Err(CompileError::NativeStackTooLarge); + } sub(cb, C_SP_REG, C_SP_REG, A64Opnd::new_uimm(slot_offset)); } } @@ -981,25 +1207,25 @@ impl Assembler } }, Insn::Mul { left, right, out } => { - // If the next instruction is jo (jump on overflow) - match (self.insns.get(insn_idx + 1), self.insns.get(insn_idx + 2)) { - (Some(Insn::JoMul(_)), _) | - (Some(Insn::PosMarker(_)), Some(Insn::JoMul(_))) => { + // If the next instruction is JoMul with RShift created by arm64_scratch_split + match (insns.get(insn_idx + 1), insns.get(insn_idx + 2)) { + (Some(Insn::RShift { out: out_sign, opnd: out_opnd, shift: out_shift }), Some(Insn::JoMul(_))) => { // Compute the high 64 bits - smulh(cb, Self::SCRATCH0, left.into(), right.into()); + smulh(cb, Self::EMIT_OPND, left.into(), right.into()); // Compute the low 64 bits // This may clobber one of the input registers, // so we do it after smulh mul(cb, out.into(), left.into(), right.into()); - // Produce a register that is all zeros or all ones - // Based on the sign bit of the 64-bit mul result - asr(cb, Self::SCRATCH1, out.into(), A64Opnd::UImm(63)); + // Insert the shift instruction created by arm64_scratch_split + // to prepare the register that has the sign bit of the high 64 bits after mul. + asr(cb, out_sign.into(), out_opnd.into(), out_shift.into()); + insn_idx += 1; // skip the next Insn::RShift // If the high 64-bits are not all zeros or all ones, // matching the sign bit, then we have an overflow - cmp(cb, Self::SCRATCH0, Self::SCRATCH1); + cmp(cb, Self::EMIT_OPND, out_sign.into()); // Insn::JoMul will emit_conditional_jump::<{Condition::NE}> } _ => { @@ -1028,87 +1254,56 @@ impl Assembler Insn::LShift { opnd, shift, out } => { lsl(cb, out.into(), opnd.into(), shift.into()); }, - store_insn @ Insn::Store { dest, src } => { - // With minor exceptions, as long as `dest` is a Mem, all forms of `src` are - // accepted. As a rule of thumb, avoid using Assembler::SCRATCH as a memory - // base register to gurantee things will work. - let &Opnd::Mem(Mem { num_bits: dest_num_bits, base: MemBase::Reg(base_reg_no), disp }) = dest else { - panic!("Unexpected Insn::Store destination in arm64_emit: {dest:?}"); - }; - - // This kind of tricky clobber can only happen for explicit use of SCRATCH_REG, - // so we panic to get the author to change their code. - #[track_caller] - fn assert_no_clobber(store_insn: &Insn, user_use: u8, backend_use: Reg) { - assert_ne!( - backend_use.reg_no, - user_use, - "Emitting {store_insn:?} would clobber {user_use:?}, in conflict with its semantics" - ); - } - - // Split src into SCRATCH0 if necessary + Insn::Store { dest, src } => { + // Split src into EMIT0_OPND if necessary let src_reg: A64Reg = match src { Opnd::Reg(reg) => *reg, // Use zero register when possible Opnd::UImm(0) | Opnd::Imm(0) => XZR_REG, // Immediates &Opnd::Imm(imm) => { - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH0_REG); - emit_load_value(cb, Self::SCRATCH0, imm as u64); - Self::SCRATCH0_REG + emit_load_value(cb, Self::EMIT_OPND, imm as u64); + Self::EMIT_REG } &Opnd::UImm(imm) => { - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH0_REG); - emit_load_value(cb, Self::SCRATCH0, imm); - Self::SCRATCH0_REG + emit_load_value(cb, Self::EMIT_OPND, imm); + Self::EMIT_REG } &Opnd::Value(value) => { - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH0_REG); - emit_load_gc_value(cb, &mut gc_offsets, Self::SCRATCH0, value); - Self::SCRATCH0_REG + emit_load_gc_value(cb, &mut gc_offsets, Self::EMIT_OPND, value); + Self::EMIT_REG } src_mem @ &Opnd::Mem(Mem { num_bits: src_num_bits, base: MemBase::Reg(src_base_reg_no), disp: src_disp }) => { - // For mem-to-mem store, load the source into SCRATCH0 - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH0_REG); + // For mem-to-mem store, load the source into EMIT0_OPND let src_mem = if mem_disp_fits_bits(src_disp) { src_mem.into() } else { - // Split the load address into SCRATCH0 first if necessary - assert_no_clobber(store_insn, src_base_reg_no, Self::SCRATCH0_REG); - load_effective_address(cb, Self::SCRATCH0, src_base_reg_no, src_disp); - A64Opnd::new_mem(dest_num_bits, Self::SCRATCH0, 0) + // Split the load address into EMIT0_OPND first if necessary + load_effective_address(cb, Self::EMIT_OPND, src_base_reg_no, src_disp); + A64Opnd::new_mem(dest.rm_num_bits(), Self::EMIT_OPND, 0) }; + let dst = A64Opnd::Reg(Self::EMIT_REG.with_num_bits(src_num_bits)); match src_num_bits { - 64 | 32 => ldur(cb, Self::SCRATCH0, src_mem), - 16 => ldurh(cb, Self::SCRATCH0, src_mem), - 8 => ldurb(cb, Self::SCRATCH0, src_mem), + 64 | 32 => ldur(cb, dst, src_mem), + 16 => ldurh(cb, dst, src_mem), + 8 => ldurb(cb, dst, src_mem), num_bits => panic!("unexpected num_bits: {num_bits}") }; - Self::SCRATCH0_REG + Self::EMIT_REG } src @ (Opnd::Mem(_) | Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during arm64_emit: {src:?}") }; let src = A64Opnd::Reg(src_reg); - // Split dest into SCRATCH1 if necessary. - let dest = if mem_disp_fits_bits(disp) { - dest.into() - } else { - assert_no_clobber(store_insn, src_reg.reg_no, Self::SCRATCH1_REG); - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH1_REG); - load_effective_address(cb, Self::SCRATCH1, base_reg_no, disp); - A64Opnd::new_mem(dest_num_bits, Self::SCRATCH1, 0) - }; - // This order may be surprising but it is correct. The way // the Arm64 assembler works, the register that is going to // be stored is first and the address is second. However in // our IR we have the address first and the register second. - match dest_num_bits { - 64 | 32 => stur(cb, src.into(), dest.into()), - 16 => sturh(cb, src.into(), dest.into()), - num_bits => panic!("unexpected dest num_bits: {} (src: {:#?}, dest: {:#?})", num_bits, src, dest), + match dest.rm_num_bits() { + 64 | 32 => stur(cb, src, dest.into()), + 16 => sturh(cb, src, dest.into()), + 8 => sturb(cb, src, dest.into()), + num_bits => panic!("unexpected dest num_bits: {num_bits} (src: {src:?}, dest: {dest:?})"), } }, Insn::Load { opnd, out } | @@ -1128,7 +1323,7 @@ impl Assembler 64 | 32 => ldur(cb, out.into(), opnd.into()), 16 => ldurh(cb, out.into(), opnd.into()), 8 => ldurb(cb, out.into(), opnd.into()), - num_bits => panic!("unexpected num_bits: {}", num_bits) + num_bits => panic!("unexpected num_bits: {num_bits}"), }; }, Opnd::Value(value) => { @@ -1151,7 +1346,6 @@ impl Assembler _ => unreachable!() }; }, - Insn::ParallelMov { .. } => unreachable!("{insn:?} should have been lowered at alloc_regs()"), Insn::Mov { dest, src } => { // This supports the following two kinds of immediates: // * The value fits into a single movz instruction @@ -1170,16 +1364,37 @@ impl Assembler let &Opnd::Mem(Mem { num_bits: _, base: MemBase::Reg(base_reg_no), disp }) = opnd else { panic!("Unexpected Insn::Lea operand in arm64_emit: {opnd:?}"); }; - load_effective_address(cb, out.into(), base_reg_no, disp); + let out_reg_no = out.unwrap_reg().reg_no; + assert_ne!(31, out_reg_no, "Lea sp, [sp, #imm] not always encodable. Use add/sub instead."); + + let out = A64Opnd::from(out); + let base_reg = A64Opnd::Reg(A64Reg { num_bits: 64, reg_no: base_reg_no }); + if ShiftedImmediate::try_from(disp.unsigned_abs() as u64).is_ok() { + // Use ADD/SUB if the displacement fits + add(cb, out, base_reg, A64Opnd::new_imm(disp.into())); + } else { + // Use a scratch reg for `out += displacement` + let disp_reg = if out_reg_no == base_reg_no { + Self::EMIT_OPND + } else { + out + }; + // Use add_extended() to interpret reg_no=31 as sp + // since the base register is never the zero register. + // Careful! Only the first two operands can refer to sp. + emit_load_value(cb, disp_reg, disp as u64); + add_extended(cb, out, base_reg, disp_reg); + } } Insn::LeaJumpTarget { out, target, .. } => { if let Target::Label(label_idx) = target { // Set output to the raw address of the label cb.label_ref(*label_idx, 4, |cb, end_addr, dst_addr| { - adr(cb, Self::SCRATCH0, A64Opnd::new_imm(dst_addr - (end_addr - 4))); + adr(cb, Self::EMIT_OPND, A64Opnd::new_imm(dst_addr - (end_addr - 4))); + Ok(()) }); - mov(cb, out.into(), Self::SCRATCH0); + mov(cb, out.into(), Self::EMIT_OPND); } else { // Set output to the jump target's raw address let target_code = target.unwrap_code_ptr(); @@ -1190,46 +1405,49 @@ impl Assembler Insn::CPush(opnd) => { emit_push(cb, opnd.into()); }, + Insn::CPushPair(opnd0, opnd1) => { + let first_push = if let Opnd::UImm(0) | Opnd::Imm(0) = opnd0 { X31 } else { opnd0.into() }; + let second_push = if let Opnd::UImm(0) | Opnd::Imm(0) = opnd1 { X31 } else { opnd1.into() }; + // Second operand ends up at the lower stack address + stp_pre(cb, second_push, first_push, A64Opnd::new_mem(64, C_SP_REG, -C_SP_STEP)); + }, Insn::CPop { out } => { emit_pop(cb, out.into()); }, Insn::CPopInto(opnd) => { emit_pop(cb, opnd.into()); }, - Insn::CPushAll => { - let regs = Assembler::get_caller_save_regs(); - - for reg in regs { - emit_push(cb, A64Opnd::Reg(reg)); - } - - // Push the flags/state register - mrs(cb, Self::SCRATCH0, SystemRegister::NZCV); - emit_push(cb, Self::SCRATCH0); - }, - Insn::CPopAll => { - let regs = Assembler::get_caller_save_regs(); - - // Pop the state/flags register - msr(cb, SystemRegister::NZCV, Self::SCRATCH0); - emit_pop(cb, Self::SCRATCH0); - - for reg in regs.into_iter().rev() { - emit_pop(cb, A64Opnd::Reg(reg)); + Insn::CPopPairInto(opnd0, opnd1) => { + let mut first_pop = opnd0.into(); + let second_pop = opnd1.into(); + // Avoid illegal load pair into the same register + // by sinking the first pop to the zero register. + if first_pop == second_pop { + first_pop = X31; } + // First operand is popped from the lower stack address + ldp_post(cb, first_pop, second_pop, A64Opnd::new_mem(64, C_SP_REG, C_SP_STEP)); }, Insn::CCall { fptr, .. } => { - // The offset to the call target in bytes - let src_addr = cb.get_write_ptr().raw_ptr(cb) as i64; - let dst_addr = *fptr as i64; - - // Use BL if the offset is short enough to encode as an immediate. - // Otherwise, use BLR with a register. - if b_offset_fits_bits((dst_addr - src_addr) / 4) { - bl(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); - } else { - emit_load_value(cb, Self::SCRATCH0, dst_addr as u64); - blr(cb, Self::SCRATCH0); + match fptr { + Opnd::UImm(fptr) => { + // The offset to the call target in bytes + let src_addr = cb.get_write_ptr().raw_ptr(cb) as i64; + let dst_addr = *fptr as i64; + + // Use BL if the offset is short enough to encode as an immediate. + // Otherwise, use BLR with a register. + if b_offset_fits_bits((dst_addr - src_addr) / 4) { + bl(cb, InstructionOffset::from_bytes((dst_addr - src_addr) as i32)); + } else { + emit_load_value(cb, Self::EMIT_OPND, dst_addr as u64); + blr(cb, Self::EMIT_OPND); + } + } + Opnd::Reg(_) => { + blr(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") } }, Insn::CRet { .. } => { @@ -1250,44 +1468,61 @@ impl Assembler emit_jmp_ptr(cb, dst_ptr, true); }, Target::Label(label_idx) => { - // Here we're going to save enough space for - // ourselves and then come back and write the - // instruction once we know the offset. We're going - // to assume we can fit into a single b instruction. - // It will panic otherwise. + // Reserve space for a single B instruction cb.label_ref(label_idx, 4, |cb, src_addr, dst_addr| { - let bytes: i32 = (dst_addr - (src_addr - 4)).try_into().unwrap(); - b(cb, InstructionOffset::from_bytes(bytes)); + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } + }); + }, + Target::Block(ref edge) => { + let label = self.block_label(edge.target); + cb.label_ref(label, 4, |cb, src_addr, dst_addr| { + // +1 since src_addr is after the instruction while A64 + // counts the offset relative to the start. + let offset = (dst_addr - src_addr) / 4 + 1; + if b_offset_fits_bits(offset) { + b(cb, InstructionOffset::from_insns(offset as i32)); + Ok(()) + } else { + Err(()) + } }); }, Target::SideExit { .. } => { - unreachable!("Target::SideExit should have been compiled by compile_side_exits") + unreachable!("Target::SideExit should have been compiled by compile_exits") }, }; }, Insn::Je(target) | Insn::Jz(target) => { - emit_conditional_jump::<{Condition::EQ}>(cb, target.clone()); + emit_conditional_jump::<{Condition::EQ}>(self, cb, target.clone()); }, Insn::Jne(target) | Insn::Jnz(target) | Insn::JoMul(target) => { - emit_conditional_jump::<{Condition::NE}>(cb, target.clone()); + emit_conditional_jump::<{Condition::NE}>(self, cb, target.clone()); }, Insn::Jl(target) => { - emit_conditional_jump::<{Condition::LT}>(cb, target.clone()); + emit_conditional_jump::<{Condition::LT}>(self, cb, target.clone()); }, Insn::Jg(target) => { - emit_conditional_jump::<{Condition::GT}>(cb, target.clone()); + emit_conditional_jump::<{Condition::GT}>(self, cb, target.clone()); }, Insn::Jge(target) => { - emit_conditional_jump::<{Condition::GE}>(cb, target.clone()); + emit_conditional_jump::<{Condition::GE}>(self, cb, target.clone()); }, Insn::Jbe(target) => { - emit_conditional_jump::<{Condition::LS}>(cb, target.clone()); + emit_conditional_jump::<{Condition::LS}>(self, cb, target.clone()); }, Insn::Jb(target) => { - emit_conditional_jump::<{Condition::CC}>(cb, target.clone()); + emit_conditional_jump::<{Condition::CC}>(self, cb, target.clone()); }, Insn::Jo(target) => { - emit_conditional_jump::<{Condition::VS}>(cb, target.clone()); + emit_conditional_jump::<{Condition::VS}>(self, cb, target.clone()); }, Insn::Joz(opnd, target) => { emit_cmp_zero_jump(cb, opnd.into(), true, target.clone()); @@ -1295,7 +1530,7 @@ impl Assembler Insn::Jonz(opnd, target) => { emit_cmp_zero_jump(cb, opnd.into(), false, target.clone()); }, - Insn::PatchPoint(_) | + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in arm64_scratch_split"), Insn::PadPatchPoint => { // If patch points are too close to each other or the end of the block, fill nop instructions if let Some(last_patch_pos) = last_patch_pos { @@ -1306,25 +1541,31 @@ impl Assembler last_patch_pos = Some(cb.get_write_pos()); }, Insn::IncrCounter { mem, value } => { - let label = cb.new_label("incr_counter_loop".to_string()); - cb.write_label(label); + // Get the status register allocated by arm64_scratch_split + let Some(Insn::Cmp { + left: status_reg @ Opnd::Reg(_), + right: Opnd::UImm(_) | Opnd::Imm(_), + }) = insns.get(insn_idx + 1) else { + panic!("arm64_scratch_split should add Cmp after IncrCounter: {:?}", insns.get(insn_idx + 1)); + }; - ldaxr(cb, Self::SCRATCH0, mem.into()); - add(cb, Self::SCRATCH0, Self::SCRATCH0, value.into()); + // Attempt to increment a counter + ldaxr(cb, Self::EMIT_OPND, mem.into()); + add(cb, Self::EMIT_OPND, Self::EMIT_OPND, value.into()); // The status register that gets used to track whether or // not the store was successful must be 32 bytes. Since we - // store the SCRATCH registers as their 64-bit versions, we + // store the EMIT registers as their 64-bit versions, we // need to rewrap it here. - let status = A64Opnd::Reg(Self::SCRATCH1.unwrap_reg().with_num_bits(32)); - stlxr(cb, status, Self::SCRATCH0, mem.into()); - - cmp(cb, Self::SCRATCH1, A64Opnd::new_uimm(0)); - emit_conditional_jump::<{Condition::NE}>(cb, Target::Label(label)); + let status = A64Opnd::Reg(status_reg.unwrap_reg().with_num_bits(32)); + stlxr(cb, status, Self::EMIT_OPND, mem.into()); }, Insn::Breakpoint => { brk(cb, A64Opnd::None); }, + Insn::Abort => { + udf(cb, u16::MAX); + }, Insn::CSelZ { truthy, falsy, out } | Insn::CSelE { truthy, falsy, out } => { csel(cb, out.into(), truthy.into(), falsy.into(), Condition::EQ); @@ -1345,7 +1586,6 @@ impl Assembler Insn::CSelGE { truthy, falsy, out } => { csel(cb, out.into(), truthy.into(), falsy.into(), Condition::GE); } - Insn::LiveReg { .. } => (), // just a reg alloc signal, no code }; insn_idx += 1; @@ -1353,90 +1593,212 @@ impl Assembler // Error if we couldn't write out everything if cb.has_dropped_bytes() { - None + Err(CompileError::OutOfMemory) } else { // No bytes dropped, so the pos markers point to valid code for (insn_idx, pos) in pos_markers { - if let Insn::PosMarker(callback) = self.insns.get(insn_idx).unwrap() { + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { callback(pos, cb); } else { panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); } } - Some(gc_offsets) + Ok(gc_offsets) } } /// Optimize and compile the stored instructions - pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Option<(CodePtr, Vec<CodePtr>)> { - let asm = self.arm64_split(); - let mut asm = asm.alloc_regs(regs)?; - asm.compile_side_exits(); - - // Create label instances in the code block - for (idx, name) in asm.label_names.iter().enumerate() { - let label = cb.new_label(name.to_string()); - assert_eq!(label, Label(idx)); - } + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_reg = !self.accept_scratch_reg; + asm_dump!(self, init); - let start_ptr = cb.get_write_ptr(); - let gc_offsets = asm.arm64_emit(cb); + let mut asm = trace_compile_phase("split", || self.arm64_split()); - if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { - cb.link_labels(); + asm_dump!(asm, split); - // Invalidate icache for newly written out region so we don't run stale code. - unsafe { rb_zjit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; + trace_compile_phase("regalloc", || { + trace_compile_phase("number_instructions", || asm.number_instructions(0)); - Some((start_ptr, gc_offsets)) - } else { - cb.clear_labels(); + let live_in = trace_compile_phase("analyze_liveness", || asm.analyze_liveness()); + let intervals = trace_compile_phase("build_intervals", || asm.build_intervals(live_in)); - None - } - } -} + // Dump live intervals if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::live_intervals) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + } + } + + let preferred_registers = trace_compile_phase("preferred_registers", || asm.preferred_register_assignments(&intervals)); + let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); + + let total_stack_slots = asm.stack_base_idx + num_stack_slots; + if total_stack_slots > Self::MAX_FRAME_STACK_SLOTS { + return Err(CompileError::NativeStackTooLarge); + } -/// LIR Instructions that are lowered to an instruction that have 2 input registers and an output -/// register can look to merge with a succeeding `Insn::Mov`. -/// For example: -/// -/// Add out, a, b -/// Mov c, out -/// -/// Can become: -/// -/// Add c, a, b -/// -/// If a, b, and c are all registers. -fn merge_three_reg_mov( - live_ranges: &Vec<LiveRange>, - iterator: &mut std::iter::Peekable<impl Iterator<Item = (usize, Insn)>>, - left: &Opnd, - right: &Opnd, - out: &mut Opnd, -) { - if let (Opnd::Reg(_) | Opnd::VReg{..}, - Opnd::Reg(_) | Opnd::VReg{..}, - Some((mov_idx, Insn::Mov { dest, src }))) - = (left, right, iterator.peek()) { - if out == src && live_ranges[out.vreg_idx()].end() == *mov_idx && matches!(*dest, Opnd::Reg(_) | Opnd::VReg{..}) { - *out = *dest; - iterator.next(); // Pop merged Insn::Mov + // Dump vreg-to-physical-register mapping if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::alloc_regs) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + + println!("VReg assignments:"); + for (i, alloc) in assignments.iter().enumerate() { + if let Some(alloc) = alloc { + let range = &intervals[i].range; + let alloc_str = match alloc { + Allocation::Reg(n) => format!("{}", regs[*n]), + Allocation::Fixed(reg) => format!("{}", reg), + Allocation::Stack(n) => format!("Stack[{}]", n), + }; + println!(" v{} => {} (range: {:?}..{:?})", i, alloc_str, range.start, range.end); + } + } + } + } + + // Update FrameSetup slot_count to account for: + // 1) stack slots reserved for block params (stack_base_idx), and + // 2) register allocator spills (num_stack_slots). + trace_compile_phase("count_stack_slots", || { + for block in asm.basic_blocks.iter_mut() { + for insn in block.insns.iter_mut() { + if let Insn::FrameSetup { slot_count, .. } = insn { + *slot_count = total_stack_slots; + } + } + } + }); + + trace_compile_phase("resolve_ssa", || { + asm.handle_caller_saved_regs(&intervals, &assignments, &C_ARG_REGREGS); + asm.resolve_ssa(&intervals, &assignments); + }); + + Ok(()) + })?; + asm_dump!(asm, alloc_regs); + + // We are moved out of SSA after resolve_ssa + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + // Exit code is compiled into a separate list of instructions that we append + // to the last reachable block before scratch_split, so it gets linearized and split. + trace_compile_phase("compile_exits", || { + let exit_insns = asm.compile_exits(); + + // Append exit instructions to the last reachable block so they are + // included in linearize_instructions and processed by scratch_split. + if let Some(&last_block) = asm.block_order().last() { + for insn in exit_insns { + asm.basic_blocks[last_block.0].insns.push(insn); + asm.basic_blocks[last_block.0].insn_ids.push(None); + } + } + }); + asm_dump!(asm, compile_exits); + + if use_scratch_reg { + asm = trace_compile_phase("scratch_split", || asm.arm64_scratch_split()); + asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = trace_compile_phase("resolve_parallel_mov", || asm.resolve_parallel_mov_pass()); + asm_dump!(asm, resolve_parallel_mov); } + + trace_compile_phase("emit", || { + // Create label instances in the code block + for (idx, name) in asm.label_names.iter().enumerate() { + let label = cb.new_label(name.to_string()); + assert_eq!(label, Label(idx)); + } + + let start_ptr = cb.get_write_ptr(); + let gc_offsets = asm.arm64_emit(cb).inspect_err(|_| cb.clear_labels())?; + assert!(!cb.has_dropped_bytes(), "emit should not drop bytes without error"); + + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; + + // Invalidate icache for newly written out region so we don't run stale code. + unsafe { rb_jit_icache_invalidate(start_ptr.raw_ptr(cb) as _, cb.get_write_ptr().raw_ptr(cb) as _) }; + + Ok((start_ptr, gc_offsets)) + }) } } #[cfg(test)] mod tests { + #[cfg(feature = "disasm")] + use crate::disasms_with; + use crate::{assert_disasm_snapshot, hexdumps}; + use super::*; - use crate::assertions::assert_disasm; + use insta::assert_snapshot; static TEMP_REGS: [Reg; 5] = [X1_REG, X9_REG, X10_REG, X14_REG, X15_REG]; fn setup_asm() -> (Assembler, CodeBlock) { - (Assembler::new(), CodeBlock::new_dummy()) + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy()) + } + + fn setup_asm_with_scratch_reg() -> (Assembler, CodeBlock, Opnd) { + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy(), scratch_reg) + } + + #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); + asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.push_insn(Insn::Je(label)); + asm.frame_teardown(JIT_PRESERVED_REGS); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @" + test(): + bb0(): + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, x19, x21, x20 + v0 = Add x19, 0x40 + Store [x21 + 0x10], v0 + Joz Exit(Interrupt), v0 + Mov x0, w0 + Mov x1, [x21 - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[x20 + 0x10], VReg32(v1) + Je bb0 + FrameTeardown x19, x21, x20 + CRet v0 + FrameTeardown x19, x21, x20 + "); } #[test] @@ -1447,11 +1809,34 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), out); asm.compile_with_num_regs(&mut cb, 2); - assert_disasm!(cb, "600080d2207d009be10300aa", {" - 0x0: mov x0, #3 - 0x4: mul x0, x9, x0 - 0x8: mov x1, x0 - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #3 + 0x4: mul x1, x9, x0 + "); + assert_snapshot!(cb.hexdump(), @"600080d2217d009b"); + } + + #[test] + fn test_conditional_branch_to_label() { + let (mut asm, mut cb) = setup_asm(); + let start = asm.new_label("start"); + let forward = asm.new_label("forward"); + + let value = asm.load(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0)); + asm.write_label(start.clone()); + asm.cmp(value, 0.into()); + asm.jg(forward.clone()); + asm.push_insn(Insn::Jl(start.clone())); + asm.write_label(forward); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [sp] + 0x4: cmp x0, #0 + 0x8: b.gt #0x10 + 0xc: b.lt #4 + "); + assert_snapshot!(cb.hexdump(), @"e00340f81f0000f14c000054cbffff54"); } #[test] @@ -1465,10 +1850,11 @@ mod tests { asm.mov(sp, new_sp); asm.compile_with_num_regs(&mut cb, 2); - assert_disasm!(cb, "ff830091ff8300d1", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add sp, sp, #0x20 0x4: sub sp, sp, #0x20 "); + assert_snapshot!(cb.hexdump(), @"ff830091ff8300d1"); } #[test] @@ -1480,10 +1866,11 @@ mod tests { asm.add_into(Opnd::Reg(X20_REG), 0x20.into()); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "ff230091948200b1", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add sp, sp, #8 0x4: adds x20, x20, #0x20 "); + assert_snapshot!(cb.hexdump(), @"ff230091948200b1"); } #[test] @@ -1494,11 +1881,12 @@ mod tests { asm.load_into(Opnd::Reg(X1_REG), difference); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "000180d2000005ebe10300aa", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #8 0x4: subs x0, x0, x5 0x8: mov x1, x0 "); + assert_snapshot!(cb.hexdump(), @"000180d2000005ebe10300aa"); } #[test] @@ -1509,10 +1897,11 @@ mod tests { asm.cret(ret_val); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "000040f8c0035fd6", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x0, [x0] 0x4: ret "); + assert_snapshot!(cb.hexdump(), @"000040f8c0035fd6"); } #[test] @@ -1521,10 +1910,14 @@ mod tests { let opnd = asm.add(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); - asm.compile_with_regs(&mut cb, vec![X3_REG]); + asm.compile_with_regs(&mut cb, vec![X3_REG]).unwrap(); // Assert that only 2 instructions were written. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: adds x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001ab400000f8"); } #[test] @@ -1536,94 +1929,102 @@ mod tests { // Testing that we pad the string to the nearest 4-byte boundary to make // it easier to jump over. - assert_eq!(16, cb.get_write_pos()); - } - - #[test] - fn test_emit_cpush_all() { - let (mut asm, mut cb) = setup_asm(); - - asm.cpush_all(); - asm.compile_with_num_regs(&mut cb, 0); - } - - #[test] - fn test_emit_cpop_all() { - let (mut asm, mut cb) = setup_asm(); - - asm.cpop_all(); - asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldnp d8, d25, [x10, #-0x140] + 0x4: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x8: .byte 0x6f, 0x72, 0x6c, 0x64 + 0xc: udf #0x21 + "); + assert_snapshot!(cb.hexdump(), @"48656c6c6f2c20776f726c6421000000"); } #[test] fn test_emit_frame() { let (mut asm, mut cb) = setup_asm(); - asm.frame_setup(&[], 0); + asm.frame_setup(&[]); asm.frame_teardown(&[]); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: mov sp, x29 + 0xc: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"fd7bbfa9fd030091bf030091fd7bc1a8"); } #[test] fn frame_setup_and_teardown() { - const THREE_REGS: &'static [Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG)]; + const THREE_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG)]; // Test 3 preserved regs (odd), odd slot_count - { + let cb1 = { let (mut asm, mut cb) = setup_asm(); - asm.frame_setup(THREE_REGS, 3); + asm.stack_base_idx = 3; + asm.frame_setup(THREE_REGS); asm.frame_teardown(THREE_REGS); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "fd7bbfa9fd030091f44fbfa9f5831ff8ff8300d1b44f7fa9b5835ef8bf030091fd7bc1a8", " - 0x0: stp x29, x30, [sp, #-0x10]! - 0x4: mov x29, sp - 0x8: stp x20, x19, [sp, #-0x10]! - 0xc: stur x21, [sp, #-8] - 0x10: sub sp, sp, #0x20 - 0x14: ldp x20, x19, [x29, #-0x10] - 0x18: ldur x21, [x29, #-0x18] - 0x1c: mov sp, x29 - 0x20: ldp x29, x30, [sp], #0x10 - "); - } + cb + }; // Test 3 preserved regs (odd), even slot_count - { + let cb2 = { let (mut asm, mut cb) = setup_asm(); - asm.frame_setup(THREE_REGS, 4); + asm.stack_base_idx = 4; + asm.frame_setup(THREE_REGS); asm.frame_teardown(THREE_REGS); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "fd7bbfa9fd030091f44fbfa9f5831ff8ffc300d1b44f7fa9b5835ef8bf030091fd7bc1a8", " - 0x0: stp x29, x30, [sp, #-0x10]! - 0x4: mov x29, sp - 0x8: stp x20, x19, [sp, #-0x10]! - 0xc: stur x21, [sp, #-8] - 0x10: sub sp, sp, #0x30 - 0x14: ldp x20, x19, [x29, #-0x10] - 0x18: ldur x21, [x29, #-0x18] - 0x1c: mov sp, x29 - 0x20: ldp x29, x30, [sp], #0x10 - "); - } + cb + }; // Test 4 preserved regs (even), odd slot_count - { - static FOUR_REGS: &'static [Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG), Opnd::Reg(X22_REG)]; + let cb3 = { + static FOUR_REGS: &[Opnd] = &[Opnd::Reg(X19_REG), Opnd::Reg(X20_REG), Opnd::Reg(X21_REG), Opnd::Reg(X22_REG)]; let (mut asm, mut cb) = setup_asm(); - asm.frame_setup(FOUR_REGS, 3); + asm.stack_base_idx = 3; + asm.frame_setup(FOUR_REGS); asm.frame_teardown(FOUR_REGS); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "fd7bbfa9fd030091f44fbfa9f657bfa9ff8300d1b44f7fa9b6577ea9bf030091fd7bc1a8", " - 0x0: stp x29, x30, [sp, #-0x10]! - 0x4: mov x29, sp - 0x8: stp x20, x19, [sp, #-0x10]! - 0xc: stp x22, x21, [sp, #-0x10]! - 0x10: sub sp, sp, #0x20 - 0x14: ldp x20, x19, [x29, #-0x10] - 0x18: ldp x22, x21, [x29, #-0x20] - 0x1c: mov sp, x29 - 0x20: ldp x29, x30, [sp], #0x10 - "); - } + cb + }; + + assert_disasm_snapshot!(disasms_with!("\n", cb1, cb2, cb3), @" + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stur x21, [sp, #-8] + 0x10: sub sp, sp, #0x30 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldur x21, [x29, #-0x18] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + + 0x0: stp x29, x30, [sp, #-0x10]! + 0x4: mov x29, sp + 0x8: stp x20, x19, [sp, #-0x10]! + 0xc: stp x22, x21, [sp, #-0x10]! + 0x10: sub sp, sp, #0x20 + 0x14: ldp x20, x19, [x29, #-0x10] + 0x18: ldp x22, x21, [x29, #-0x20] + 0x1c: mov sp, x29 + 0x20: ldp x29, x30, [sp], #0x10 + "); + assert_snapshot!(hexdumps!(cb1, cb2, cb3), @" + fd7bbfa9fd030091f44fbfa9f5831ff8ff8300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f5831ff8ffc300d1b44f7fa9b5835ef8bf030091fd7bc1a8 + fd7bbfa9fd030091f44fbfa9f657bfa9ff8300d1b44f7fa9b6577ea9bf030091fd7bc1a8 + "); } #[test] @@ -1632,8 +2033,18 @@ mod tests { let target: CodePtr = cb.get_write_ptr().add_bytes(80); - asm.je(Target::CodePtr(target)); + asm.push_insn(Insn::Je(Target::CodePtr(target))); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: b.eq #0x50 + 0x4: nop + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"800200541f2003d51f2003d51f2003d51f2003d51f2003d5"); } #[test] @@ -1643,8 +2054,18 @@ mod tests { let offset = 1 << 21; let target: CodePtr = cb.get_write_ptr().add_bytes(offset); - asm.je(Target::CodePtr(target)); + asm.push_insn(Insn::Je(Target::CodePtr(target))); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: b.ne #8 + 0x4: b #0x200000 + 0x8: nop + 0xc: nop + 0x10: nop + 0x14: nop + "); + assert_snapshot!(cb.hexdump(), @"41000054ffff07141f2003d51f2003d51f2003d51f2003d5"); } #[test] @@ -1662,8 +2083,8 @@ mod tests { } asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "e07b40b2e063208b000180d22000a0f2e063208b000083d2e063208be0230891e02308d1e0ff8292e063208b00ff9fd2c0ffbff2e0ffdff2e0fffff2e063208be08361b2e063208b", " - 0x0: orr x0, xzr, #0x7fffffff + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0x7fffffff 0x4: add x0, sp, x0 0x8: mov x0, #8 0xc: movk x0, #1, lsl #16 @@ -1679,9 +2100,29 @@ mod tests { 0x34: movk x0, #0xffff, lsl #32 0x38: movk x0, #0xffff, lsl #48 0x3c: add x0, sp, x0 - 0x40: orr x0, xzr, #0xffffffff80000000 + 0x40: mov x0, #-0x80000000 0x44: add x0, sp, x0 "); + assert_snapshot!(cb.hexdump(), @"e07b40b2e063208b000180d22000a0f2e063208b000083d2e063208be0230891e02308d1e0ff8292e063208b00ff9fd2c0ffbff2e0ffdff2e0fffff2e063208be08361b2e063208b"); + } + + #[test] + fn test_load_larg_disp_mem() { + let (mut asm, mut cb) = setup_asm(); + + let extended_ivars = asm.load(Opnd::mem(64, NATIVE_STACK_PTR, 0)); + let result = asm.load(Opnd::mem(VALUE_BITS, extended_ivars, 1000 * SIZEOF_VALUE_I32)); + asm.store(Opnd::mem(VALUE_BITS, NATIVE_STACK_PTR, 0), result); + + asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [sp] + 0x4: mov x16, #0x1f40 + 0x8: add x0, x0, x16, uxtx + 0xc: ldur x0, [x0] + 0x10: stur x0, [sp] + "); + assert_snapshot!(cb.hexdump(), @"e00340f810e883d20060308b000040f8e00300f8"); } #[test] @@ -1696,18 +2137,19 @@ mod tests { asm.store(large_mem, large_mem); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "f0170cd1100240f8100000f8100040f8f1170cd1300200f8f0170cd1100240f8f1170cd1300200f8", " - 0x0: sub x16, sp, #0x305 - 0x4: ldur x16, [x16] - 0x8: stur x16, [x0] - 0xc: ldur x16, [x0] - 0x10: sub x17, sp, #0x305 - 0x14: stur x16, [x17] - 0x18: sub x16, sp, #0x305 - 0x1c: ldur x16, [x16] - 0x20: sub x17, sp, #0x305 - 0x24: stur x16, [x17] + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: sub x17, sp, #0x305 + 0x4: ldur x16, [x17] + 0x8: stur x16, [x0] + 0xc: sub x15, sp, #0x305 + 0x10: ldur x16, [x0] + 0x14: stur x16, [x15] + 0x18: sub x15, sp, #0x305 + 0x1c: sub x17, sp, #0x305 + 0x20: ldur x16, [x17] + 0x24: stur x16, [x15] "); + assert_snapshot!(cb.hexdump(), @"f1170cd1300240f8100000f8ef170cd1100040f8f00100f8ef170cd1f1170cd1300240f8f00100f8"); } #[test] @@ -1720,24 +2162,54 @@ mod tests { // Side exit code are compiled without the split pass, so we directly call emit here to // emulate that scenario. + for name in &asm.label_names { + cb.new_label(name.to_string()); + } let gc_offsets = asm.arm64_emit(&mut cb).unwrap(); assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); - assert_disasm!(cb, "50000058030000140010000000000000b00200f8", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldr x16, #8 0x4: b #0x10 - 0x8: .byte 0x00, 0x10, 0x00, 0x00 - 0xc: .byte 0x00, 0x00, 0x00, 0x00 + 0x8: udf #0x1000 + 0xc: udf #0 0x10: stur x16, [x21] "); + assert_snapshot!(cb.hexdump(), @"50000058030000140010000000000000b00200f8"); + } + + #[test] + fn test_store_with_valid_scratch_reg() { + let (mut asm, mut cb, scratch_reg) = setup_asm_with_scratch_reg(); + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0x3902 + 0x4: movk x16, #8, lsl #16 + 0x8: stur x16, [x15] + "); + assert_snapshot!(cb.hexdump(), @"502087d21001a0f2f00100f8"); + } + + #[test] + #[should_panic] + fn test_store_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); + let (mut asm, mut cb) = setup_asm(); + // This would put the source into scratch_reg, messing up the destination + asm.store(Opnd::mem(64, scratch_reg, 0), 0x83902.into()); + + asm.compile_with_num_regs(&mut cb, 0); } #[test] #[should_panic] - fn test_store_unserviceable() { + fn test_load_into_with_invalid_scratch_reg() { + let (_, scratch_reg) = Assembler::new_with_scratch_reg(); let (mut asm, mut cb) = setup_asm(); - // This would put the source into SCRATCH_REG, messing up the destination - asm.store(Opnd::mem(64, Opnd::Reg(Assembler::SCRATCH_REG), 0), 0x83902.into()); + // This would put the source into scratch_reg, messing up the destination + asm.load_into(scratch_reg, 0x83902.into()); asm.compile_with_num_regs(&mut cb, 0); } @@ -1754,6 +2226,16 @@ mod tests { asm.store(Opnd::mem(64, SP, 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: adr x16, #8 + 0x4: mov x0, x16 + 0x8: ldnp d8, d25, [x10, #-0x140] + 0xc: .byte 0x6f, 0x2c, 0x20, 0x77 + 0x10: .byte 0x6f, 0x72, 0x6c, 0x64 + 0x14: udf #0x21 + 0x18: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"50000010e00310aa48656c6c6f2c20776f726c6421000000a00200f8"); } #[test] @@ -1765,7 +2247,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that two instructions were written: LDUR and STUR. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur x0, [x21] + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a00240f8a00200f8"); } #[test] @@ -1777,7 +2263,12 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that three instructions were written: ADD, LDUR, and STUR. - assert_eq!(12, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x21, #0x400 + 0x4: ldur x0, [x0] + 0x8: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"a0021091000040f8a00200f8"); } #[test] @@ -1789,7 +2280,13 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that three instructions were written: MOVZ, ADD, LDUR, and STUR. - assert_eq!(16, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0x1001 + 0x4: add x0, x21, x0, uxtx + 0x8: ldur x0, [x0] + 0xc: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"200082d2a062208b000040f8a00200f8"); } #[test] @@ -1802,7 +2299,11 @@ mod tests { // Assert that only two instructions were written since the value is an // immediate. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #4 + 0x4: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"800080d2a00200f8"); } #[test] @@ -1815,7 +2316,14 @@ mod tests { // Assert that five instructions were written since the value is not an // immediate and needs to be loaded into a register. - assert_eq!(20, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldr x0, #8 + 0x4: b #0x10 + 0x8: eon x0, x0, x30, ror #0 + 0xc: eon x30, x23, x30, ror #50 + 0x10: stur x0, [x21] + "); + assert_snapshot!(cb.hexdump(), @"40000058030000140000fecafecafecaa00200f8"); } #[test] @@ -1826,6 +2334,12 @@ mod tests { // All ones is not encodable with a bitmask immediate, // so this needs one register asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #0xffffffff + 0x4: tst w0, w0 + "); + assert_snapshot!(cb.hexdump(), @"e07f40b21f00006a"); } #[test] @@ -1834,6 +2348,9 @@ mod tests { let w0 = Opnd::Reg(X0_REG).with_num_bits(32); asm.test(w0, Opnd::UImm(0x80000001)); asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst w0, #0x80000001"); + assert_snapshot!(cb.hexdump(), @"1f040172"); } #[test] @@ -1843,6 +2360,12 @@ mod tests { let opnd = asm.or(Opnd::Reg(X0_REG), Opnd::Reg(X1_REG)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: orr x0, x0, x1 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"000001aa400000f8"); } #[test] @@ -1852,6 +2375,12 @@ mod tests { let opnd = asm.lshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: lsl x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00e87bd3400000f8"); } #[test] @@ -1861,6 +2390,12 @@ mod tests { let opnd = asm.rshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: asr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc4593400000f8"); } #[test] @@ -1870,6 +2405,12 @@ mod tests { let opnd = asm.urshift(Opnd::Reg(X0_REG), Opnd::UImm(5)); asm.store(Opnd::mem(64, Opnd::Reg(X2_REG), 0), opnd); asm.compile_with_num_regs(&mut cb, 1); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: lsr x0, x0, #5 + 0x4: stur x0, [x2] + "); + assert_snapshot!(cb.hexdump(), @"00fc45d3400000f8"); } #[test] @@ -1880,7 +2421,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 0); // Assert that only one instruction was written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, x1"); + assert_snapshot!(cb.hexdump(), @"1f0001ea"); } #[test] @@ -1891,7 +2433,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 0); // Assert that only one instruction was written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); } #[test] @@ -1902,7 +2445,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that a load and a test instruction were written. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); } #[test] @@ -1913,7 +2460,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 0); // Assert that only one instruction was written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #7"); + assert_snapshot!(cb.hexdump(), @"1f0840f2"); } #[test] @@ -1924,7 +2472,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that a load and a test instruction were written. - assert_eq!(8, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #5 + 0x4: tst x0, x0 + "); + assert_snapshot!(cb.hexdump(), @"a00080d21f0000ea"); } #[test] @@ -1935,7 +2487,8 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); // Assert that a test instruction is written. - assert_eq!(4, cb.get_write_pos()); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: tst x0, #-7"); + assert_snapshot!(cb.hexdump(), @"1ff47df2"); } #[test] @@ -1945,6 +2498,13 @@ mod tests { let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); asm.cmp(shape_opnd, Opnd::UImm(4097)); asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldur w0, [x0, #6] + 0x4: mov x1, #0x1001 + 0x8: cmp w0, w1 + "); + assert_snapshot!(cb.hexdump(), @"006040b8210082d21f00016b"); } #[test] @@ -1954,6 +2514,12 @@ mod tests { let shape_opnd = Opnd::mem(16, Opnd::Reg(X0_REG), 0); asm.store(shape_opnd, Opnd::UImm(4097)); asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0x1001 + 0x4: sturh w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"300082d210000078"); } #[test] @@ -1963,6 +2529,12 @@ mod tests { let shape_opnd = Opnd::mem(32, Opnd::Reg(X0_REG), 6); asm.store(shape_opnd, Opnd::UImm(4097)); asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #0x1001 + 0x4: stur w16, [x0, #6] + "); + assert_snapshot!(cb.hexdump(), @"300082d2106000b8"); } #[test] @@ -1974,10 +2546,11 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "000001ca400000f8", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: eor x0, x0, x1 0x4: stur x0, [x2] "); + assert_snapshot!(cb.hexdump(), @"000001ca400000f8"); } #[test] @@ -2011,9 +2584,8 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::mem(64, CFP, 8)); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "618240f8", {" - 0x0: ldur x1, [x19, #8] - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: ldur x1, [x19, #8]"); + assert_snapshot!(cb.hexdump(), @"618240f8"); } #[test] @@ -2024,10 +2596,11 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::UImm(0x10000)); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "e1ff9fd2e10370b2", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x1, #0xffff - 0x4: orr x1, xzr, #0x10000 - "}); + 0x4: mov x1, #0x10000 + "); + assert_snapshot!(cb.hexdump(), @"e1ff9fd2e10370b2"); } #[test] @@ -2038,11 +2611,42 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), out); asm.compile_with_num_regs(&mut cb, 2); - assert_disasm!(cb, "800280d2010080d201b0819a", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x0, #0x14 0x4: mov x1, #0 0x8: csel x1, x0, x1, lt - "}); + "); + assert_snapshot!(cb.hexdump(), @"800280d2010080d201b0819a"); + } + + #[test] + fn test_exceeding_label_branch_generate_bounds() { + // The immediate in a conditional branch is a 19 bit unsigned integer + // which has a max value of 2^18 - 1. + const IMMEDIATE_MAX_VALUE: usize = 2usize.pow(18) - 1; + + // `IMMEDIATE_MAX_VALUE` number of dummy instructions will be generated + // plus a compare, a jump instruction, and a label. + // Adding page_size to avoid OOM on the last page. + let page_size = unsafe { rb_jit_get_page_size() } as usize; + let memory_required = (IMMEDIATE_MAX_VALUE + 8) * 4 + page_size; + + crate::options::rb_zjit_prepare_options(); // Allow `get_option!` in Assembler + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + let mut cb = CodeBlock::new_dummy_sized(memory_required); + + let far_label = asm.new_label("far"); + + asm.cmp(Opnd::Reg(X0_REG), Opnd::UImm(1)); + asm.push_insn(Insn::Je(far_label.clone())); + + (0..IMMEDIATE_MAX_VALUE).for_each(|_| { + asm.mov(Opnd::Reg(TEMP_REGS[0]), Opnd::Reg(TEMP_REGS[2])); + }); + + asm.write_label(far_label.clone()); + assert_eq!(Err(CompileError::LabelLinkingFailure), asm.compile(&mut cb)); } #[test] @@ -2054,15 +2658,29 @@ mod tests { asm.mov(Opnd::Reg(TEMP_REGS[0]), out); asm.compile_with_num_regs(&mut cb, 2); - assert_disasm!(cb, "200500b1010400b1", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: adds x0, x9, #1 0x4: adds x1, x0, #1 - "}); + "); + assert_snapshot!(cb.hexdump(), @"200500b1010400b1"); + } + + #[test] + fn test_store_spilled_byte() { + let (mut asm, mut cb) = setup_asm(); + + asm.store(Opnd::mem(8, C_RET_OPND, 0), Opnd::mem(8, C_RET_OPND, 8)); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: ldurb w16, [x0, #8] + 0x4: sturb w16, [x0] + "); + assert_snapshot!(cb.hexdump(), @"1080403810000038"); } #[test] - fn test_reorder_c_args_no_cycle() { - crate::options::rb_zjit_prepare_options(); + fn test_ccall_resolve_parallel_moves_no_cycle() { let (mut asm, mut cb) = setup_asm(); asm.ccall(0 as _, vec![ @@ -2071,15 +2689,15 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "100080d200023fd6", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov x16, #0 0x4: blr x16 - "}); + "); + assert_snapshot!(cb.hexdump(), @"100080d200023fd6"); } #[test] - fn test_reorder_c_args_single_cycle() { - crate::options::rb_zjit_prepare_options(); + fn test_ccall_resolve_parallel_moves_single_cycle() { let (mut asm, mut cb) = setup_asm(); // x0 and x1 form a cycle @@ -2090,18 +2708,18 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "f00300aae00301aae10310aa100080d200023fd6", {" - 0x0: mov x16, x0 + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 0x4: mov x0, x1 - 0x8: mov x1, x16 + 0x8: mov x1, x15 0xc: mov x16, #0 0x10: blr x16 - "}); + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae1030faa100080d200023fd6"); } #[test] - fn test_reorder_c_args_two_cycles() { - crate::options::rb_zjit_prepare_options(); + fn test_ccall_resolve_parallel_moves_two_cycles() { let (mut asm, mut cb) = setup_asm(); // x0 and x1 form a cycle, and x2 and rcx form another cycle @@ -2113,21 +2731,89 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "f00302aae20303aae30310aaf00300aae00301aae10310aa100080d200023fd6", {" - 0x0: mov x16, x2 - 0x4: mov x2, x3 - 0x8: mov x3, x16 - 0xc: mov x16, x0 - 0x10: mov x0, x1 - 0x14: mov x1, x16 - 0x18: mov x16, #0 - 0x1c: blr x16 - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 + 0x4: mov x0, x1 + 0x8: mov x1, x15 + 0xc: mov x15, x2 + 0x10: mov x2, x3 + 0x14: mov x3, x15 + 0x18: mov x16, #0 + 0x1c: blr x16 + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae1030faaef0302aae20303aae3030faa100080d200023fd6"); + } + + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: stp x1, x0, [sp, #-0x10]! + 0x14: stp x3, x2, [sp, #-0x10]! + 0x18: mov x16, #0 + 0x1c: blr x16 + 0x20: ldp x3, x2, [sp], #0x10 + 0x24: ldp x1, x0, [sp], #0x10 + 0x28: adds x0, x0, x1 + 0x2c: adds x0, x2, x3 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2e103bfa9e30bbfa9100080d200023fd6e30bc1a8e103c1a8000001ab400003ab"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: mov x2, #3 + 0xc: mov x3, #4 + 0x10: mov x4, #5 + 0x14: stp x1, x0, [sp, #-0x10]! + 0x18: stp x3, x2, [sp, #-0x10]! + 0x1c: stp xzr, x4, [sp, #-0x10]! + 0x20: mov x16, #0 + 0x24: blr x16 + 0x28: ldp xzr, x4, [sp], #0x10 + 0x2c: ldp x3, x2, [sp], #0x10 + 0x30: ldp x1, x0, [sp], #0x10 + 0x34: adds x0, x0, x1 + 0x38: adds x0, x2, x3 + 0x3c: adds x0, x2, x4 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2620080d2830080d2a40080d2e103bfa9e30bbfa9ff13bfa9100080d200023fd6ff13c1a8e30bc1a8e103c1a8000001ab400003ab400004ab"); } #[test] - fn test_reorder_c_args_large_cycle() { - crate::options::rb_zjit_prepare_options(); + fn test_ccall_resolve_parallel_moves_large_cycle() { let (mut asm, mut cb) = setup_asm(); // x0, x1, and x2 form a cycle @@ -2138,14 +2824,106 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "f00300aae00301aae10302aae20310aa100080d200023fd6", {" - 0x0: mov x16, x0 + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x15, x0 0x4: mov x0, x1 0x8: mov x1, x2 - 0xc: mov x2, x16 + 0xc: mov x2, x15 0x10: mov x16, #0 0x14: blr x16 - "}); + "); + assert_snapshot!(cb.hexdump(), @"ef0300aae00301aae10302aae2030faa100080d200023fd6"); } + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: stp x1, x0, [sp, #-0x10]! + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e103bfa9"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x0, #1 + 0x4: mov x1, #2 + 0x8: ldp x0, x1, [sp], #0x10 + "); + assert_snapshot!(cb.hexdump(), @"200080d2410080d2e007c1a8"); + } + + #[test] + fn test_split_spilled_lshift() { + let (mut asm, mut cb) = setup_asm(); + + let opnd_vreg = asm.load(1.into()); + let out_vreg = asm.lshift(opnd_vreg, Opnd::UImm(1)); + asm.mov(C_RET_OPND, out_vreg); + asm.compile_with_num_regs(&mut cb, 0); // spill every VReg + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov x16, #1 + 0x4: stur x16, [x29, #-8] + 0x8: ldur x15, [x29, #-8] + 0xc: lsl x0, x15, #1 + "); + assert_snapshot!(cb.hexdump(), @"300080d2b0831ff8af835ff8e0f97fd3"); + } + + #[test] + fn test_split_load16_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(16, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x0, #0x200 + 0x4: ldurh w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"0000089100004078"); + } + + #[test] + fn test_split_load32_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(32, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x0, #0x200 + 0x4: ldur w0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040b8"); + } + + #[test] + fn test_split_load64_mem_mem_with_large_displacement() { + let (mut asm, mut cb) = setup_asm(); + + let _ = asm.load(Opnd::mem(64, C_RET_OPND, 0x200)); + asm.compile(&mut cb).unwrap(); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: add x0, x0, #0x200 + 0x4: ldur x0, [x0] + "); + assert_snapshot!(cb.hexdump(), @"00000891000040f8"); + } } diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 1bb4cd024b..91a1a3ffcf 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -1,34 +1,236 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt; use std::mem::take; -use crate::codegen::local_size_and_idx_to_ep_offset; -use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32}; -use crate::hir::SideExitReason; -use crate::options::{debug, get_option}; +use std::rc::Rc; +use crate::bitset::BitSet; +use crate::codegen::{local_size_and_idx_to_ep_offset, perf_symbol_range_start, perf_symbol_range_end}; +use crate::cruby::{IseqPtr, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary, YarvInsnIdx }; +use crate::hir::{Invariant, SideExitReason}; +use crate::hir; +use crate::options::{TraceExits, PerfMap, get_option}; use crate::cruby::VALUE; +use crate::payload::{IseqVersionRef, get_or_create_iseq_payload}; +use crate::stats::{exit_counter_ptr, exit_counter_ptr_for_opcode, side_exit_counter, CompileError}; use crate::virtualmem::CodePtr; use crate::asm::{CodeBlock, Label}; +use crate::state::{ZJITState, rb_zjit_record_exit_stack}; + +/// LIR Block ID. Unique ID for each block, and also defined in LIR so +/// we can differentiate it from HIR block ids. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct BlockId(pub usize); + +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct VRegId(pub usize); + +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 + } +} + +impl From<VRegId> for usize { + fn from(val: VRegId) -> Self { + val.0 + } +} + +impl std::fmt::Display for BlockId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "l{}", self.0) + } +} + +impl std::fmt::Display for VRegId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "v{}", self.0) + } +} + +/// Dummy HIR block ID used when creating test or invalid LIR blocks +const DUMMY_HIR_BLOCK_ID: usize = usize::MAX; +/// Dummy RPO index used when creating test or invalid LIR blocks +const DUMMY_RPO_INDEX: usize = usize::MAX; + +/// LIR Instruction ID. Unique ID for each instruction in the LIR. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] +pub struct InsnId(pub usize); + +impl From<InsnId> for usize { + fn from(val: InsnId) -> Self { + val.0 + } +} + +impl std::fmt::Display for InsnId { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "i{}", self.0) + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct BranchEdge { + pub target: BlockId, + pub args: Vec<Opnd>, +} + +#[derive(Clone, Debug)] +pub struct BasicBlock { + // Unique id for this block + pub id: BlockId, + + // HIR block this LIR block was lowered from. Not injective: multiple LIR blocks may share + // the same hir_block_id because we split HIR blocks into multiple LIR blocks during lowering. + pub hir_block_id: hir::BlockId, + + pub is_entry: bool, + + // Instructions in this basic block + pub insns: Vec<Insn>, + + // Instruction IDs for each instruction (same length as insns) + pub insn_ids: Vec<Option<InsnId>>, + + // Input parameters for this block + pub parameters: Vec<Opnd>, + + // RPO position of the source HIR block + pub rpo_index: usize, + + // Range of instruction IDs in this block + pub from: InsnId, + pub to: InsnId, +} + +pub struct EdgePair(Option<BranchEdge>, Option<BranchEdge>); + +impl BasicBlock { + fn new(id: BlockId, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> Self { + Self { + id, + hir_block_id, + is_entry, + insns: vec![], + insn_ids: vec![], + parameters: vec![], + rpo_index, + from: InsnId(0), + to: InsnId(0), + } + } + + pub fn is_dummy(&self) -> bool { + self.hir_block_id == hir::BlockId(DUMMY_HIR_BLOCK_ID) + } + + pub fn add_parameter(&mut self, param: Opnd) { + self.parameters.push(param); + } + + pub fn push_insn(&mut self, insn: Insn) { + self.insns.push(insn); + self.insn_ids.push(None); + } + + pub fn edges(&self) -> EdgePair { + // Stub blocks (from new_block_without_id) have no real CFG structure. + if self.rpo_index == DUMMY_RPO_INDEX { + return EdgePair(None, None); + } + assert!(self.insns.last().unwrap().is_terminator()); + let extract_edge = |insn: &Insn| -> Option<BranchEdge> { + if let Some(Target::Block(edge)) = insn.target() { + Some(edge.clone()) + } else { + None + } + }; + + match self.insns.as_slice() { + [] => panic!("empty block"), + [.., second_last, last] => { + EdgePair(extract_edge(second_last), extract_edge(last)) + }, + [.., last] => { + EdgePair(extract_edge(last), None) + } + } + } + + /// Sort key for scheduling blocks in code layout order + pub fn sort_key(&self) -> (usize, usize) { + (self.rpo_index, self.id.0) + } + + pub fn successors(&self) -> Vec<BlockId> { + let EdgePair(edge1, edge2) = self.edges(); + let mut succs = Vec::new(); + if let Some(edge) = edge1 { + succs.push(edge.target); + } + if let Some(edge) = edge2 { + succs.push(edge.target); + } + succs + } + + /// Get the output VRegs for this block. + /// These are VRegs referenced by operands passed to successor blocks via block edges. + /// This function is used for live range calculations and should _not_ + /// be used for parallel moves between blocks + pub fn out_vregs(&self) -> Vec<VRegId> { + let EdgePair(edge1, edge2) = self.edges(); + let mut out_vregs = Vec::new(); + if let Some(edge) = edge1 { + for arg in &edge.args { + for idx in arg.vreg_ids() { + out_vregs.push(idx); + } + } + } + if let Some(edge) = edge2 { + for arg in &edge.args { + for idx in arg.vreg_ids() { + out_vregs.push(idx); + } + } + } + out_vregs + } +} pub use crate::backend::current::{ + mem_base_reg, Reg, EC, CFP, SP, - NATIVE_STACK_PTR, NATIVE_BASE_PTR, - C_ARG_OPNDS, C_RET_REG, C_RET_OPND, + NATIVE_BASE_PTR, + C_ARG_OPNDS, C_RET_OPND, }; -pub const SCRATCH_OPND: Opnd = Opnd::Reg(Assembler::SCRATCH_REG); -pub static JIT_PRESERVED_REGS: &'static [Opnd] = &[CFP, SP, EC]; +pub static JIT_PRESERVED_REGS: &[Opnd] = &[CFP, SP, EC]; // Memory operand base -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash, Ord, PartialOrd)] pub enum MemBase { + /// Register: Every Opnd::Mem should have MemBase::Reg as of emit. Reg(u8), - VReg(usize), + /// Virtual register: Lowered to MemBase::Reg or MemBase::Stack during register assignment. + VReg(VRegId), + /// Stack slot: a direct stack access. `stack_membase_to_mem()` turns this + /// into `[NATIVE_BASE_PTR + disp]`, so scratch splitting can use it as a + /// normal memory operand without first loading a pointer from the stack. + Stack { stack_idx: usize, num_bits: u8 }, + /// A pointer stored in a stack slot, used as a memory base. + /// Unlike Stack, this first loads the pointer value from the stack slot + /// into a scratch register, then uses that register as the base for the + /// memory access with the Mem's displacement. + /// Created when a VReg used as MemBase is spilled to the stack. + StackIndirect { stack_idx: usize }, } // Memory location -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] pub struct Mem { // Base register number or instruction index @@ -41,6 +243,31 @@ pub struct Mem pub num_bits: u8, } +impl fmt::Display for Mem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.num_bits != 64 { + write!(f, "Mem{}", self.num_bits)?; + } + write!(f, "[")?; + match self.base { + MemBase::Reg(reg_no) => write!(f, "{}", mem_base_reg(reg_no))?, + MemBase::VReg(idx) => write!(f, "{idx}")?, + MemBase::Stack { stack_idx, num_bits } if num_bits == 64 => write!(f, "Stack[{stack_idx}]")?, + MemBase::StackIndirect { stack_idx } => write!(f, "*Stack[{stack_idx}]")?, + MemBase::Stack { stack_idx, num_bits } => write!(f, "Stack{num_bits}[{stack_idx}]")?, + } + if self.disp != 0 { + let sign = if self.disp > 0 { '+' } else { '-' }; + write!(f, " {sign} ")?; + if self.disp.abs() >= 10 { + write!(f, "0x")?; + } + write!(f, "{:x}", self.disp.abs())?; + } + write!(f, "]") + } +} + impl fmt::Debug for Mem { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!(fmt, "Mem{}[{:?}", self.num_bits, self.base)?; @@ -54,7 +281,7 @@ impl fmt::Debug for Mem { } /// Operand to an IR instruction -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] pub enum Opnd { None, // For insns with no output @@ -62,8 +289,8 @@ pub enum Opnd // Immediate Ruby value, may be GC'd, movable Value(VALUE), - /// Virtual register. Lowered to Reg or Mem in Assembler::alloc_regs(). - VReg{ idx: usize, num_bits: u8 }, + /// Virtual register. Lowered to Reg or Mem during register assignment. + VReg{ idx: VRegId, num_bits: u8 }, // Low-level operands, for lowering Imm(i64), // Raw signed immediate @@ -72,14 +299,67 @@ pub enum Opnd Reg(Reg), // Machine register } +impl PartialOrd for Opnd { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for Opnd { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn case_order(opnd: &Opnd) -> u8 { + match opnd { + Opnd::None => 0, + Opnd::Value(_) => 1, + Opnd::VReg { .. } => 2, + Opnd::Imm(_) => 3, + Opnd::UImm(_) => 4, + Opnd::Mem(_) => 5, + Opnd::Reg(_) => 6, + } + } + match (self, other) { + (Opnd::None, Opnd::None) => std::cmp::Ordering::Equal, + (Opnd::Value(l), Opnd::Value(r)) => l.0.cmp(&r.0), + (Opnd::VReg { idx: lidx, num_bits: lnum_bits }, Opnd::VReg { idx: ridx, num_bits: rnum_bits }) => (lidx, lnum_bits).cmp(&(ridx, rnum_bits)), + (Opnd::Imm(l), Opnd::Imm(r)) => l.cmp(&r), + (Opnd::UImm(l), Opnd::UImm(r)) => l.cmp(&r), + (Opnd::Mem(l), Opnd::Mem(r)) => l.cmp(&r), + (Opnd::Reg(l), Opnd::Reg(r)) => l.cmp(&r), + (l, r) => { + case_order(l).cmp(&case_order(r)) + } + } + } +} + +impl fmt::Display for Opnd { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Opnd::*; + match self { + None => write!(f, "None"), + Value(VALUE(value)) if *value < 10 => write!(f, "Value({value:x})"), + Value(VALUE(value)) => write!(f, "Value(0x{value:x})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(f, "{idx}"), + VReg { idx, num_bits } => write!(f, "VReg{num_bits}({idx})"), + Imm(value) if value.abs() < 10 => write!(f, "Imm({value:x})"), + Imm(value) => write!(f, "Imm(0x{value:x})"), + UImm(value) if *value < 10 => write!(f, "{value:x}"), + UImm(value) => write!(f, "0x{value:x}"), + Mem(mem) => write!(f, "{mem}"), + Reg(reg) => write!(f, "{reg}"), + } + } +} + impl fmt::Debug for Opnd { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { use Opnd::*; match self { Self::None => write!(fmt, "None"), Value(val) => write!(fmt, "Value({val:?})"), - VReg { idx, num_bits } if *num_bits == 64 => write!(fmt, "VReg({idx})"), - VReg { idx, num_bits } => write!(fmt, "VReg{num_bits}({idx})"), + VReg { idx, num_bits } if *num_bits == 64 => write!(fmt, "VReg({})", idx.0), + VReg { idx, num_bits } => write!(fmt, "VReg{num_bits}({})", idx.0), Imm(signed) => write!(fmt, "{signed:x}_i64"), UImm(unsigned) => write!(fmt, "{unsigned:x}_u64"), // Say Mem and Reg only once @@ -91,6 +371,11 @@ impl fmt::Debug for Opnd { impl Opnd { + /// Returns true if this operand is a virtual register + pub fn is_vreg(&self) -> bool { + matches!(self, Opnd::VReg { .. }) + } + /// Convenience constructor for memory operands pub fn mem(num_bits: u8, base: Opnd, disp: i32) -> Self { match base { @@ -98,8 +383,8 @@ impl Opnd assert!(base_reg.num_bits == 64); Opnd::Mem(Mem { base: MemBase::Reg(base_reg.reg_no), - disp: disp, - num_bits: num_bits, + disp, + num_bits, }) }, @@ -107,8 +392,8 @@ impl Opnd assert!(num_bits <= out_num_bits); Opnd::Mem(Mem { base: MemBase::VReg(idx), - disp: disp, - num_bits: num_bits, + disp, + num_bits, }) }, @@ -130,13 +415,25 @@ impl Opnd } /// Unwrap the index of a VReg - pub fn vreg_idx(&self) -> usize { + pub fn vreg_idx(&self) -> VRegId { match self { Opnd::VReg { idx, .. } => *idx, _ => unreachable!("trying to unwrap {self:?} into VReg"), } } + /// Extract VReg indices from this operand, including memory base VRegs. + /// Returns an iterator over all VRegIds referenced by this operand. + pub fn vreg_ids(&self) -> impl Iterator<Item = VRegId> { + let mut ids = [None, None]; + match self { + Opnd::VReg { idx, .. } => { ids[0] = Some(*idx); } + Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { ids[0] = Some(*idx); } + _ => {} + } + ids.into_iter().flatten() + } + /// Get the size in bits for this operand if there is one. pub fn num_bits(&self) -> Option<u8> { match *self { @@ -169,10 +466,10 @@ impl Opnd pub fn map_index(self, indices: &[usize]) -> Opnd { match self { Opnd::VReg { idx, num_bits } => { - Opnd::VReg { idx: indices[idx], num_bits } + Opnd::VReg { idx: VRegId(indices[idx.0]), num_bits } } Opnd::Mem(Mem { base: MemBase::VReg(idx), disp, num_bits }) => { - Opnd::Mem(Mem { base: MemBase::VReg(indices[idx]), disp, num_bits }) + Opnd::Mem(Mem { base: MemBase::VReg(VRegId(indices[idx.0])), disp, num_bits }) }, _ => self } @@ -246,27 +543,72 @@ impl From<VALUE> for Opnd { } } +/// Context for a side exit. If `SideExit` matches, it reuses the same code. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SideExit { + pub pc: Opnd, + pub stack: Vec<Opnd>, + pub locals: Vec<Opnd>, + pub iseq: IseqPtr, + /// If set, the side exit will call the recompile function with these arguments + /// to profile the send and invalidate the ISEQ for recompilation. + pub recompile: Option<SideExitRecompile>, +} + +/// Arguments for the recompile callback on side exit. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SideExitRecompile { + pub iseq: Opnd, + pub insn_idx: u32, + pub strategy: hir::Recompile, +} + /// Branch target (something that we can jump to) /// for branch instructions -#[derive(Clone, Debug)] +#[derive(Clone)] pub enum Target { /// Pointer to a piece of ZJIT-generated code CodePtr(CodePtr), /// A label within the generated code Label(Label), + /// An LIR branch edge + Block(BranchEdge), /// Side exit to the interpreter SideExit { - pc: *const VALUE, - stack: Vec<Opnd>, - locals: Vec<Opnd>, - /// We use this to enrich asm comments. + /// Context used for compiling the side exit + exit: SideExit, + /// We use this to increment exit counters reason: SideExitReason, - /// Some if the side exit should write this label. We use it for patch points. - label: Option<Label>, }, } +impl fmt::Debug for Target { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Target::CodePtr(ptr) => write!(f, "CodePtr({:?})", ptr), + Target::Label(label) => write!(f, "Label({:?})", label), + Target::Block(edge) => { + if edge.args.is_empty() { + write!(f, "Block({:?})", edge.target) + } else { + write!(f, "Block({:?}(", edge.target)?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{:?}", arg)?; + } + write!(f, "))") + } + } + Target::SideExit { exit, reason } => { + write!(f, "SideExit {{ exit: {:?}, reason: {:?} }}", exit, reason) + } + } + } +} + impl Target { pub fn unwrap_label(&self) -> Label { @@ -290,9 +632,10 @@ impl From<CodePtr> for Target { } } -type PosMarkerFn = Box<dyn Fn(CodePtr, &CodeBlock)>; +type PosMarkerFn = Rc<dyn Fn(CodePtr, &CodeBlock)>; /// ZJIT Low-level IR instruction +#[derive(Clone)] pub enum Insn { /// Add two operands together, and return the result as a new operand. Add { left: Opnd, right: Opnd, out: Opnd }, @@ -308,6 +651,9 @@ pub enum Insn { #[allow(dead_code)] Breakpoint, + // Abort the process + Abort, + /// Add a comment into the IR at the point that this instruction is added. /// It won't have any impact on that actual compiled code. Comment(String), @@ -318,29 +664,33 @@ pub enum Insn { /// Pop a register from the C stack CPop { out: Opnd }, - /// Pop all of the caller-save registers and the flags from the C stack - CPopAll, - /// Pop a register from the C stack and store it into another register CPopInto(Opnd), + /// Pop a pair of registers from the C stack and store it into a pair of registers. + /// The registers are popped from left to right. + CPopPairInto(Opnd, Opnd), + /// Push a register onto the C stack CPush(Opnd), - /// Push all of the caller-save registers and the flags to the C stack - CPushAll, + /// Push a pair of registers onto the C stack. + /// The registers are pushed from left to right. + CPushPair(Opnd, Opnd), // C function call with N arguments (variadic) CCall { opnds: Vec<Opnd>, - fptr: *const u8, + /// The function pointer to be called. This should be Opnd::const_ptr + /// (Opnd::UImm) in most cases. gen_entry_trampoline() uses Opnd::Reg. + fptr: Opnd, /// Optional PosMarker to remember the start address of the C call. /// It's embedded here to insert the PosMarker after push instructions - /// that are split from this CCall on alloc_regs(). + /// that are split from this CCall during register assignment. start_marker: Option<PosMarkerFn>, /// Optional PosMarker to remember the end address of the C call. /// It's embedded here to insert the PosMarker before pop instructions - /// that are split from this CCall on alloc_regs(). + /// that are split from this CCall during register assignment. end_marker: Option<PosMarkerFn>, out: Opnd, }, @@ -437,9 +787,6 @@ pub enum Insn { // Load effective address Lea { opnd: Opnd, out: Opnd }, - /// Take a specific register. Signal the register allocator to not use it. - LiveReg { opnd: Opnd, out: Opnd }, - // A low-level instruction that loads a value into a register. Load { opnd: Opnd, out: Opnd }, @@ -453,10 +800,6 @@ pub enum Insn { /// Shift a value left by a certain amount. LShift { opnd: Opnd, shift: Opnd, out: Opnd }, - /// A set of parallel moves into registers. - /// The backend breaks cycles if there are any cycles between moves. - ParallelMov { moves: Vec<(Reg, Opnd)> }, - // A low-level mov instruction. It accepts two operands. Mov { dest: Opnd, src: Opnd }, @@ -470,7 +813,7 @@ pub enum Insn { Or { left: Opnd, right: Opnd, out: Opnd }, /// Patch point that will be rewritten to a jump to a side exit on invalidation. - PatchPoint(Target), + PatchPoint { target: Target, invariant: Invariant, version: IseqVersionRef }, /// Make sure the last PatchPoint has enough space to insert a jump. /// We insert this instruction at the end of each block so that the jump @@ -503,17 +846,140 @@ pub enum Insn { Xor { left: Opnd, right: Opnd, out: Opnd } } +macro_rules! target_for_each_operand_impl { + ($self:expr, $visit_many:ident) => { + match $self { + Target::SideExit { exit: SideExit { stack, locals, .. }, .. } => { + visit_many!(stack); + visit_many!(locals); + } + Target::Block(edge) => { + visit_many!(edge.args); + } + Target::CodePtr(_) | Target::Label(_) => {} + } + } +} + +/// Macro that enumerates all operands of an Insn, dispatching to caller-provided `$visit_one` +/// macro for a single `Opnd` field and `$visit_many` macro for a slice/`Vec` of `Opnd`s. Used by +/// both `for_each_operand` and `for_each_operand_mut`. +macro_rules! for_each_operand_impl { + ($self:expr, $visit_one:ident, $visit_many:ident $(, $const:expr)?) => { + match $self { + Insn::Jbe(target) | + Insn::Jb(target) | + Insn::Je(target) | + Insn::Jl(target) | + Insn::Jg(target) | + Insn::Jge(target) | + Insn::Jmp(target) | + Insn::Jne(target) | + Insn::Jnz(target) | + Insn::Jo(target) | + Insn::JoMul(target) | + Insn::Jz(target) | + Insn::Label(target) | + Insn::LeaJumpTarget { target, .. } | + Insn::PatchPoint { target, .. } => { + target_for_each_operand_impl!(target, $visit_many); + } + Insn::Joz(opnd, target) | + Insn::Jonz(opnd, target) => { + visit_one!(opnd); + target_for_each_operand_impl!(target, $visit_many); + } + + Insn::BakeString(_) | + Insn::Breakpoint | Insn::Abort | + Insn::Comment(_) | + Insn::CPop { .. } | + Insn::PadPatchPoint | + Insn::PosMarker(_) => {}, + + Insn::CPopInto(opnd) | + Insn::CPush(opnd) | + Insn::CRet(opnd) | + Insn::JmpOpnd(opnd) | + Insn::Lea { opnd, .. } | + Insn::Load { opnd, .. } | + Insn::LoadSExt { opnd, .. } | + Insn::Not { opnd, .. } => { + visit_one!(opnd); + } + Insn::Add { left: opnd0, right: opnd1, .. } | + Insn::And { left: opnd0, right: opnd1, .. } | + Insn::CPushPair(opnd0, opnd1) | + Insn::CPopPairInto(opnd0, opnd1) | + Insn::Cmp { left: opnd0, right: opnd1 } | + Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelGE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelL { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelLE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNE { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelNZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::CSelZ { truthy: opnd0, falsy: opnd1, .. } | + Insn::IncrCounter { mem: opnd0, value: opnd1, .. } | + Insn::LoadInto { dest: opnd0, opnd: opnd1 } | + Insn::LShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Mov { dest: opnd0, src: opnd1 } | + Insn::Or { left: opnd0, right: opnd1, .. } | + Insn::RShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Store { dest: opnd0, src: opnd1 } | + Insn::Sub { left: opnd0, right: opnd1, .. } | + Insn::Mul { left: opnd0, right: opnd1, .. } | + Insn::Test { left: opnd0, right: opnd1 } | + Insn::URShift { opnd: opnd0, shift: opnd1, .. } | + Insn::Xor { left: opnd0, right: opnd1, .. } => { + visit_one!(opnd0); + visit_one!(opnd1); + } + Insn::CCall { opnds, .. } => { + visit_many!(opnds); + } + // only iterate over preserved in the const iterator + #[allow(unused_variables)] + Insn::FrameSetup { preserved, .. } | + Insn::FrameTeardown { preserved } => { + $( + visit_many!(preserved); + $const; + )? + } + } + } +} + impl Insn { - /// Create an iterator that will yield a non-mutable reference to each - /// operand in turn for this instruction. - pub(super) fn opnd_iter(&self) -> InsnOpndIterator<'_> { - InsnOpndIterator::new(self) + pub fn opnd_count(&self) -> usize { + let mut count = 0; + self.for_each_operand(|_| count += 1); + count + } + + /// Call `f` on each operand (Opnd) of this instruction. + pub fn for_each_operand(&self, mut f: impl FnMut(Opnd)) { + macro_rules! visit_one { ($id:expr) => { f(*$id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id) } }; } + // Extra () is a throw-away parameter to avoid iterating over FrameSetup/FrameTeardown + // preserved in the mutable iterator. + for_each_operand_impl!(self, visit_one, visit_many, ()); + } + + /// Call `f` on a mutable reference to each operand (Opnd) of this instruction. + pub fn for_each_operand_mut(&mut self, mut f: impl FnMut(&mut Opnd)) { + macro_rules! visit_one { ($id:expr) => { f($id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter_mut() { f(id) } }; } + for_each_operand_impl!(self, visit_one, visit_many); } - /// Create an iterator that will yield a mutable reference to each operand - /// in turn for this instruction. - pub(super) fn opnd_iter_mut(&mut self) -> InsnOpndMutIterator<'_> { - InsnOpndMutIterator::new(self) + /// Call `f` on each operand, short-circuiting on the first error. + pub fn try_for_each_operand<E>(&self, mut f: impl FnMut(Opnd) -> Result<(), E>) -> Result<(), E> { + macro_rules! visit_one { ($id:expr) => { f(*$id)? }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id)? } }; } + for_each_operand_impl!(self, visit_one, visit_many, ()); + Ok(()) } /// Get a mutable reference to a Target if it exists. @@ -535,7 +1001,7 @@ impl Insn { Insn::Jonz(_, target) | Insn::Label(target) | Insn::LeaJumpTarget { target, .. } | - Insn::PatchPoint(target) => { + Insn::PatchPoint { target, .. } => { Some(target) } _ => None, @@ -550,13 +1016,14 @@ impl Insn { Insn::And { .. } => "And", Insn::BakeString(_) => "BakeString", Insn::Breakpoint => "Breakpoint", + Insn::Abort => "Abort", Insn::Comment(_) => "Comment", Insn::Cmp { .. } => "Cmp", Insn::CPop { .. } => "CPop", - Insn::CPopAll => "CPopAll", Insn::CPopInto(_) => "CPopInto", + Insn::CPopPairInto(_, _) => "CPopPairInto", Insn::CPush(_) => "CPush", - Insn::CPushAll => "CPushAll", + Insn::CPushPair(_, _) => "CPushPair", Insn::CCall { .. } => "CCall", Insn::CRet(_) => "CRet", Insn::CSelE { .. } => "CSelE", @@ -588,16 +1055,14 @@ impl Insn { Insn::Label(_) => "Label", Insn::LeaJumpTarget { .. } => "LeaJumpTarget", Insn::Lea { .. } => "Lea", - Insn::LiveReg { .. } => "LiveReg", Insn::Load { .. } => "Load", Insn::LoadInto { .. } => "LoadInto", Insn::LoadSExt { .. } => "LoadSExt", Insn::LShift { .. } => "LShift", - Insn::ParallelMov { .. } => "ParallelMov", Insn::Mov { .. } => "Mov", Insn::Not { .. } => "Not", Insn::Or { .. } => "Or", - Insn::PatchPoint(_) => "PatchPoint", + Insn::PatchPoint { .. } => "PatchPoint", Insn::PadPatchPoint => "PadPatchPoint", Insn::PosMarker(_) => "PosMarker", Insn::RShift { .. } => "RShift", @@ -628,7 +1093,6 @@ impl Insn { Insn::CSelZ { out, .. } | Insn::Lea { out, .. } | Insn::LeaJumpTarget { out, .. } | - Insn::LiveReg { out, .. } | Insn::Load { out, .. } | Insn::LoadSExt { out, .. } | Insn::LShift { out, .. } | @@ -661,7 +1125,6 @@ impl Insn { Insn::CSelZ { out, .. } | Insn::Lea { out, .. } | Insn::LeaJumpTarget { out, .. } | - Insn::LiveReg { out, .. } | Insn::Load { out, .. } | Insn::LoadSExt { out, .. } | Insn::LShift { out, .. } | @@ -695,7 +1158,7 @@ impl Insn { Insn::Jonz(_, target) | Insn::Label(target) | Insn::LeaJumpTarget { target, .. } | - Insn::PatchPoint(target) => Some(target), + Insn::PatchPoint { target, .. } => Some(target), _ => None } } @@ -708,321 +1171,35 @@ impl Insn { _ => None } } -} - -/// An iterator that will yield a non-mutable reference to each operand in turn -/// for the given instruction. -pub(super) struct InsnOpndIterator<'a> { - insn: &'a Insn, - idx: usize, -} - -impl<'a> InsnOpndIterator<'a> { - fn new(insn: &'a Insn) -> Self { - Self { insn, idx: 0 } - } -} - -impl<'a> Iterator for InsnOpndIterator<'a> { - type Item = &'a Opnd; - - fn next(&mut self) -> Option<Self::Item> { - match self.insn { - Insn::Jbe(target) | - Insn::Jb(target) | - Insn::Je(target) | - Insn::Jl(target) | - Insn::Jg(target) | - Insn::Jge(target) | - Insn::Jmp(target) | - Insn::Jne(target) | - Insn::Jnz(target) | - Insn::Jo(target) | - Insn::JoMul(target) | - Insn::Jz(target) | - Insn::Label(target) | - Insn::LeaJumpTarget { target, .. } | - Insn::PatchPoint(target) => { - if let Target::SideExit { stack, locals, .. } = target { - let stack_idx = self.idx; - if stack_idx < stack.len() { - let opnd = &stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } - let local_idx = self.idx - stack.len(); - if local_idx < locals.len() { - let opnd = &locals[local_idx]; - self.idx += 1; - return Some(opnd); - } - } - None + /// Returns true if this instruction is a terminator (ends a basic block). + pub fn is_terminator(&self) -> bool { + self.is_jump() || + match self { + Insn::CRet(_) => true, + _ => false } - - Insn::Joz(opnd, target) | - Insn::Jonz(opnd, target) => { - if self.idx == 0 { - self.idx += 1; - return Some(opnd); - } - - if let Target::SideExit { stack, locals, .. } = target { - let stack_idx = self.idx - 1; - if stack_idx < stack.len() { - let opnd = &stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } - - let local_idx = stack_idx - stack.len(); - if local_idx < locals.len() { - let opnd = &locals[local_idx]; - self.idx += 1; - return Some(opnd); - } - } - None - } - - Insn::BakeString(_) | - Insn::Breakpoint | - Insn::Comment(_) | - Insn::CPop { .. } | - Insn::CPopAll | - Insn::CPushAll | - Insn::FrameSetup { .. } | - Insn::FrameTeardown { .. } | - Insn::PadPatchPoint | - Insn::PosMarker(_) => None, - - Insn::CPopInto(opnd) | - Insn::CPush(opnd) | - Insn::CRet(opnd) | - Insn::JmpOpnd(opnd) | - Insn::Lea { opnd, .. } | - Insn::LiveReg { opnd, .. } | - Insn::Load { opnd, .. } | - Insn::LoadSExt { opnd, .. } | - Insn::Not { opnd, .. } => { - match self.idx { - 0 => { - self.idx += 1; - Some(opnd) - }, - _ => None - } - }, - Insn::Add { left: opnd0, right: opnd1, .. } | - Insn::And { left: opnd0, right: opnd1, .. } | - Insn::Cmp { left: opnd0, right: opnd1 } | - Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelGE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelL { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelLE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelNE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelNZ { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelZ { truthy: opnd0, falsy: opnd1, .. } | - Insn::IncrCounter { mem: opnd0, value: opnd1, .. } | - Insn::LoadInto { dest: opnd0, opnd: opnd1 } | - Insn::LShift { opnd: opnd0, shift: opnd1, .. } | - Insn::Mov { dest: opnd0, src: opnd1 } | - Insn::Or { left: opnd0, right: opnd1, .. } | - Insn::RShift { opnd: opnd0, shift: opnd1, .. } | - Insn::Store { dest: opnd0, src: opnd1 } | - Insn::Sub { left: opnd0, right: opnd1, .. } | - Insn::Mul { left: opnd0, right: opnd1, .. } | - Insn::Test { left: opnd0, right: opnd1 } | - Insn::URShift { opnd: opnd0, shift: opnd1, .. } | - Insn::Xor { left: opnd0, right: opnd1, .. } => { - match self.idx { - 0 => { - self.idx += 1; - Some(opnd0) - } - 1 => { - self.idx += 1; - Some(opnd1) - } - _ => None - } - }, - Insn::CCall { opnds, .. } => { - if self.idx < opnds.len() { - let opnd = &opnds[self.idx]; - self.idx += 1; - Some(opnd) - } else { - None - } - }, - Insn::ParallelMov { moves } => { - if self.idx < moves.len() { - let opnd = &moves[self.idx].1; - self.idx += 1; - Some(opnd) - } else { - None - } - }, - } - } -} - -/// An iterator that will yield each operand in turn for the given instruction. -pub(super) struct InsnOpndMutIterator<'a> { - insn: &'a mut Insn, - idx: usize, -} - -impl<'a> InsnOpndMutIterator<'a> { - fn new(insn: &'a mut Insn) -> Self { - Self { insn, idx: 0 } } - pub(super) fn next(&mut self) -> Option<&mut Opnd> { - match self.insn { - Insn::Jbe(target) | - Insn::Jb(target) | - Insn::Je(target) | - Insn::Jl(target) | - Insn::Jg(target) | - Insn::Jge(target) | - Insn::Jmp(target) | - Insn::Jne(target) | - Insn::Jnz(target) | - Insn::Jo(target) | - Insn::JoMul(target) | - Insn::Jz(target) | - Insn::Label(target) | - Insn::LeaJumpTarget { target, .. } | - Insn::PatchPoint(target) => { - if let Target::SideExit { stack, locals, .. } = target { - let stack_idx = self.idx; - if stack_idx < stack.len() { - let opnd = &mut stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } - - let local_idx = self.idx - stack.len(); - if local_idx < locals.len() { - let opnd = &mut locals[local_idx]; - self.idx += 1; - return Some(opnd); - } - } - None - } - - Insn::Joz(opnd, target) | - Insn::Jonz(opnd, target) => { - if self.idx == 0 { - self.idx += 1; - return Some(opnd); - } - - if let Target::SideExit { stack, locals, .. } = target { - let stack_idx = self.idx - 1; - if stack_idx < stack.len() { - let opnd = &mut stack[stack_idx]; - self.idx += 1; - return Some(opnd); - } - - let local_idx = stack_idx - stack.len(); - if local_idx < locals.len() { - let opnd = &mut locals[local_idx]; - self.idx += 1; - return Some(opnd); - } - } - None - } - - Insn::BakeString(_) | - Insn::Breakpoint | - Insn::Comment(_) | - Insn::CPop { .. } | - Insn::CPopAll | - Insn::CPushAll | - Insn::FrameSetup { .. } | - Insn::FrameTeardown { .. } | - Insn::PadPatchPoint | - Insn::PosMarker(_) => None, - - Insn::CPopInto(opnd) | - Insn::CPush(opnd) | - Insn::CRet(opnd) | - Insn::JmpOpnd(opnd) | - Insn::Lea { opnd, .. } | - Insn::LiveReg { opnd, .. } | - Insn::Load { opnd, .. } | - Insn::LoadSExt { opnd, .. } | - Insn::Not { opnd, .. } => { - match self.idx { - 0 => { - self.idx += 1; - Some(opnd) - }, - _ => None - } - }, - Insn::Add { left: opnd0, right: opnd1, .. } | - Insn::And { left: opnd0, right: opnd1, .. } | - Insn::Cmp { left: opnd0, right: opnd1 } | - Insn::CSelE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelG { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelGE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelL { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelLE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelNE { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelNZ { truthy: opnd0, falsy: opnd1, .. } | - Insn::CSelZ { truthy: opnd0, falsy: opnd1, .. } | - Insn::IncrCounter { mem: opnd0, value: opnd1, .. } | - Insn::LoadInto { dest: opnd0, opnd: opnd1 } | - Insn::LShift { opnd: opnd0, shift: opnd1, .. } | - Insn::Mov { dest: opnd0, src: opnd1 } | - Insn::Or { left: opnd0, right: opnd1, .. } | - Insn::RShift { opnd: opnd0, shift: opnd1, .. } | - Insn::Store { dest: opnd0, src: opnd1 } | - Insn::Sub { left: opnd0, right: opnd1, .. } | - Insn::Mul { left: opnd0, right: opnd1, .. } | - Insn::Test { left: opnd0, right: opnd1 } | - Insn::URShift { opnd: opnd0, shift: opnd1, .. } | - Insn::Xor { left: opnd0, right: opnd1, .. } => { - match self.idx { - 0 => { - self.idx += 1; - Some(opnd0) - } - 1 => { - self.idx += 1; - Some(opnd1) - } - _ => None - } - }, - Insn::CCall { opnds, .. } => { - if self.idx < opnds.len() { - let opnd = &mut opnds[self.idx]; - self.idx += 1; - Some(opnd) - } else { - None - } - }, - Insn::ParallelMov { moves } => { - if self.idx < moves.len() { - let opnd = &mut moves[self.idx].1; - self.idx += 1; - Some(opnd) - } else { - None - } - }, + /// Returns true if this instruction is a jump. + pub fn is_jump(&self) -> bool { + match self { + Insn::Jbe(_) | + Insn::Jb(_) | + Insn::Je(_) | + Insn::Jl(_) | + Insn::Jg(_) | + Insn::Jge(_) | + Insn::Jmp(_) | + Insn::JmpOpnd(_) | + Insn::Jne(_) | + Insn::Jnz(_) | + Insn::Jo(_) | + Insn::JoMul(_) | + Insn::Jz(_) | + Insn::Joz(..) | + Insn::Jonz(..) => true, + _ => false } } } @@ -1031,14 +1208,15 @@ impl fmt::Debug for Insn { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { write!(fmt, "{}(", self.op())?; - // Print list of operands - let mut opnd_iter = self.opnd_iter(); - if let Some(first_opnd) = opnd_iter.next() { - write!(fmt, "{first_opnd:?}")?; - } - for opnd in opnd_iter { - write!(fmt, ", {opnd:?}")?; + if let Insn::FrameSetup { slot_count, .. } = self { + write!(fmt, "{slot_count}")?; } + // Print list of operands + let mut sep = ""; + self.for_each_operand(|opnd| { + write!(fmt, "{sep}{opnd:?}").unwrap(); + sep = ", "; + }); write!(fmt, ")")?; // Print text, target, and pos if they are present @@ -1057,9 +1235,9 @@ impl fmt::Debug for Insn { /// TODO: Consider supporting lifetime holes #[derive(Clone, Debug, PartialEq)] pub struct LiveRange { - /// Index of the first instruction that used the VReg (inclusive) + /// Index of the first instruction that used the VReg pub start: Option<usize>, - /// Index of the last instruction that used the VReg (inclusive) + /// Index of the last instruction that used the VReg pub end: Option<usize>, } @@ -1075,85 +1253,132 @@ impl LiveRange { } } -/// RegisterPool manages which registers are used by which VReg -struct RegisterPool { - /// List of registers that can be allocated - regs: Vec<Reg>, - - /// Some(vreg_idx) if the register at the index in `pool` is used by the VReg. - /// None if the register is not in use. - pool: Vec<Option<usize>>, - - /// The number of live registers. - /// Provides a quick way to query `pool.filter(|r| r.is_some()).count()` - live_regs: usize, +/// Live Interval of a VReg +#[derive(Clone)] +pub struct Interval { + pub range: LiveRange, + pub id: usize, } -impl RegisterPool { - /// Initialize a register pool - fn new(regs: Vec<Reg>) -> Self { - let pool = vec![None; regs.len()]; - RegisterPool { - regs, - pool, - live_regs: 0, +impl Interval { + /// Create a new Interval with no range + pub fn new(i: usize) -> Self { + Self { + range: LiveRange { + start: None, + end: None, + }, + id: i, } } - /// Mutate the pool to indicate that the register at the index - /// has been allocated and is live. - fn alloc_reg(&mut self, vreg_idx: usize) -> Option<Reg> { - for (reg_idx, reg) in self.regs.iter().enumerate() { - if self.pool[reg_idx].is_none() { - self.pool[reg_idx] = Some(vreg_idx); - self.live_regs += 1; - return Some(*reg); - } + /// Check if the interval is alive at position x + /// Panics if the range is not set + pub fn survives(&self, x: usize) -> bool { + assert!(self.range.start.is_some() && self.range.end.is_some(), "survives called on interval with no range"); + let start = self.range.start.unwrap(); + let end = self.range.end.unwrap(); + start < x && end > x + } + + pub fn born_at(&self, x:usize) -> bool { + let start = self.range.start.unwrap(); + start == x + } + + pub fn dies_at(&self, x:usize) -> bool { + let end = self.range.end.unwrap(); + end == x + } + + pub fn has_bounds(&self) -> bool { + self.range.start.is_some() && self.range.end.is_some() + } + + /// Add a range to the interval, extending it if necessary + pub fn add_range(&mut self, from: usize, to: usize) { + if to <= from { + panic!("Invalid range: {} to {}", from, to); } - None + + if self.range.start.is_none() { + self.range.start = Some(from); + self.range.end = Some(to); + return; + } + + // Extend the range to cover both the existing range and the new range + self.range.start = Some(self.range.start.unwrap().min(from)); + self.range.end = Some(self.range.end.unwrap().max(to)); } - /// Allocate a specific register - fn take_reg(&mut self, reg: &Reg, vreg_idx: usize) -> Reg { - let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no) - .unwrap_or_else(|| panic!("Unable to find register: {}", reg.reg_no)); - assert_eq!(self.pool[reg_idx], None, "register already allocated for VReg({:?})", self.pool[reg_idx]); - self.pool[reg_idx] = Some(vreg_idx); - self.live_regs += 1; - *reg + /// Set the start of the range + pub fn set_from(&mut self, from: usize) { + let end = self.range.end.unwrap_or(from); + self.range.start = Some(from); + self.range.end = Some(end); } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Allocation { + Reg(usize), + Fixed(Reg), + Stack(usize), +} + +impl Allocation { + fn assigned_reg(self) -> Option<Reg> { + use crate::backend::current::ALLOC_REGS; - // Mutate the pool to indicate that the given register is being returned - // as it is no longer used by the instruction that previously held it. - fn dealloc_reg(&mut self, reg: &Reg) { - let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no) - .unwrap_or_else(|| panic!("Unable to find register: {}", reg.reg_no)); - if self.pool[reg_idx].is_some() { - self.pool[reg_idx] = None; - self.live_regs -= 1; + match self { + Allocation::Reg(n) => Some(ALLOC_REGS[n]), + Allocation::Fixed(reg) => Some(reg), + Allocation::Stack(_) => None, } } - /// Return a list of (Reg, vreg_idx) tuples for all live registers - fn live_regs(&self) -> Vec<(Reg, usize)> { - let mut live_regs = Vec::with_capacity(self.live_regs); - for (reg_idx, ®) in self.regs.iter().enumerate() { - if let Some(vreg_idx) = self.pool[reg_idx] { - live_regs.push((reg, vreg_idx)); + fn alloc_pool_index(self, num_registers: usize) -> Option<usize> { + match self { + Allocation::Reg(n) => Some(n), + Allocation::Fixed(reg) => { + use crate::backend::current::ALLOC_REGS; + + ALLOC_REGS + .iter() + .take(num_registers) + .position(|candidate| candidate.reg_no == reg.reg_no) } + Allocation::Stack(_) => None, } - live_regs } +} + +/// StackState converts abstract stack slots into concrete stack addresses. +pub struct StackState { + /// Copy of Assembler::stack_base_idx. Used for calculating stack slot offsets. + stack_base_idx: usize, +} - /// Return vreg_idx if a given register is already in use - fn vreg_for(&self, reg: &Reg) -> Option<usize> { - let reg_idx = self.regs.iter().position(|elem| elem.reg_no == reg.reg_no).unwrap(); - self.pool[reg_idx] +impl StackState { + /// Initialize a stack allocator + pub(super) fn new(stack_base_idx: usize) -> Self { + StackState { stack_base_idx } } - /// Return true if no register is in use - fn is_empty(&self) -> bool { - self.live_regs == 0 + /// Convert a stack index to the `disp` of the stack slot + fn stack_idx_to_disp(&self, stack_idx: usize) -> i32 { + (self.stack_base_idx + stack_idx + 1) as i32 * -SIZEOF_VALUE_I32 + } + /// Convert MemBase::Stack to Mem + pub(super) fn stack_membase_to_mem(&self, membase: MemBase) -> Mem { + match membase { + MemBase::Stack { stack_idx, num_bits } => { + let disp = self.stack_idx_to_disp(stack_idx); + Mem { base: MemBase::Reg(NATIVE_BASE_PTR.unwrap_reg().reg_no), disp, num_bits } + } + _ => unreachable!(), + } } } @@ -1162,72 +1387,311 @@ const ASSEMBLER_INSNS_CAPACITY: usize = 256; /// Object into which we assemble instructions to be /// optimized and lowered +#[derive(Clone)] pub struct Assembler { - pub(super) insns: Vec<Insn>, + pub basic_blocks: Vec<BasicBlock>, - /// Live range for each VReg indexed by its `idx`` - pub(super) live_ranges: Vec<LiveRange>, + /// The block to which new instructions are added. Used during HIR to LIR lowering to + /// determine which LIR block we should add instructions to. Set by `set_current_block()` + /// and automatically set to new entry blocks created by `new_block()`. + current_block_id: BlockId, + + /// Number of VRegs allocated + pub(super) num_vregs: usize, /// Names of labels pub(super) label_names: Vec<String>, + + /// If true, `push_insn` is allowed to use scratch registers. + /// On `compile`, it also disables the backend's use of them. + pub(super) accept_scratch_reg: bool, + + /// The maximum number of stack slots that have been reserved + /// by Assembler::alloc_stack(). + pub stack_base_idx: usize, + + /// If Some, the next ccall should verify its leafness + leaf_ccall_stack_size: Option<usize>, + + /// Current instruction index, incremented for each instruction pushed + idx: usize, } impl Assembler { - /// Create an Assembler + /// Create an Assembler with defaults pub fn new() -> Self { - Self::new_with_label_names(Vec::default(), 0) + Self { + label_names: Vec::default(), + accept_scratch_reg: false, + stack_base_idx: 0, + leaf_ccall_stack_size: None, + basic_blocks: Vec::default(), + current_block_id: BlockId(0), + num_vregs: 0, + idx: 0, + } } - /// Create an Assembler with parameters that are populated by another Assembler instance. - /// This API is used for copying an Assembler for the next compiler pass. - pub fn new_with_label_names(label_names: Vec<String>, num_vregs: usize) -> Self { - let mut live_ranges = Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY); - live_ranges.resize(num_vregs, LiveRange { start: None, end: None }); + /// Create an Assembler, reserving a specified number of stack slots + pub fn new_with_stack_slots(stack_base_idx: usize) -> Self { + Self { stack_base_idx, ..Self::new() } + } - Self { - insns: Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY), - live_ranges, - label_names, + /// Create an Assembler that allows the use of scratch registers. + /// This should be called only through [`Self::new_with_scratch_reg`]. + pub(super) fn new_with_accept_scratch_reg(accept_scratch_reg: bool) -> Self { + Self { accept_scratch_reg, ..Self::new() } + } + + /// Create an Assembler with parameters of another Assembler and empty instructions. + /// Compiler passes build a next Assembler with this API and insert new instructions to it. + pub(super) fn new_with_asm(old_asm: &Assembler) -> Self { + let mut asm = Self { + label_names: old_asm.label_names.clone(), + accept_scratch_reg: old_asm.accept_scratch_reg, + stack_base_idx: old_asm.stack_base_idx, + ..Self::new() + }; + + // Initialize basic blocks from the old assembler, preserving hir_block_id and entry flag + // but with empty instruction lists + for old_block in &old_asm.basic_blocks { + asm.new_block_from_old_block(&old_block); + } + + // Initialize num_vregs to match the old assembler's size + // This allows reusing VRegs from the old assembler + asm.num_vregs = old_asm.num_vregs; + + asm + } + + // Create a new LIR basic block. Returns the newly created block ID + pub fn new_block(&mut self, hir_block_id: hir::BlockId, is_entry: bool, rpo_index: usize) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let lir_bb = BasicBlock::new(bb_id, hir_block_id, is_entry, rpo_index); + self.basic_blocks.push(lir_bb); + if is_entry { + self.set_current_block(bb_id); + } + bb_id + } + + // Create a new LIR basic block from an old one. This should only be used + // when creating new assemblers during passes when we want to translate + // one assembler to a new one. + pub fn new_block_from_old_block(&mut self, old_block: &BasicBlock) -> BlockId { + let bb_id = BlockId(self.basic_blocks.len()); + let mut lir_bb = BasicBlock::new(bb_id, old_block.hir_block_id, old_block.is_entry, old_block.rpo_index); + lir_bb.parameters = old_block.parameters.clone(); + self.basic_blocks.push(lir_bb); + bb_id + } + + // Create a LIR basic block without a valid HIR block ID (for testing or internal use). + pub fn new_block_without_id(&mut self, name: &str) -> BlockId { + let bb_id = self.new_block(hir::BlockId(DUMMY_HIR_BLOCK_ID), true, DUMMY_RPO_INDEX); + let label = self.new_label(name); + self.write_label(label); + bb_id + } + + pub fn set_current_block(&mut self, block_id: BlockId) { + self.current_block_id = block_id; + } + + pub fn current_block(&mut self) -> &mut BasicBlock { + &mut self.basic_blocks[self.current_block_id.0] + } + + /// Return basic blocks sorted by RPO index, then by block ID. + /// TODO: Use a more advanced scheduling algorithm + pub fn sorted_blocks(&self) -> Vec<&BasicBlock> { + let mut sorted: Vec<&BasicBlock> = self.basic_blocks.iter().collect(); + sorted.sort_by_key(|block| block.sort_key()); + sorted + } + + /// Validate that jump instructions only appear as the last two instructions in each block. + /// This is a CFG invariant that ensures proper control flow structure. + /// Only active in debug builds. + pub fn validate_jump_positions(&self) { + for block in &self.basic_blocks { + let insns = &block.insns; + let len = insns.len(); + + // Check all instructions except the last two + for (i, insn) in insns.iter().enumerate() { + debug_assert!( + !insn.is_terminator() || i >= len.saturating_sub(2), + "Invalid jump position in block {:?}: {:?} at position {} (block has {} instructions). \ + Jumps must only appear in the last two positions.", + block.id, insn.op(), i, len + ); + } + } + } + + /// Return true if `opnd` is or depends on `reg` + pub fn has_reg(opnd: Opnd, reg: Reg) -> bool { + match opnd { + Opnd::Reg(opnd_reg) => opnd_reg == reg, + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) => reg_no == reg.reg_no, + _ => false, + } + } + + pub fn instruction_iterator(&mut self) -> InsnIter { + let mut blocks = take(&mut self.basic_blocks); + blocks.sort_by_key(|block| block.sort_key()); + + let mut iter = InsnIter { + blocks, + current_block_idx: 0, + current_insn_iter: vec![].into_iter(), // Will be replaced immediately + peeked: None, + index: 0, + }; + + // Set up first block's iterator + if !iter.blocks.is_empty() { + iter.current_insn_iter = take(&mut iter.blocks[0].insns).into_iter(); + } + + iter + } + + pub fn linearize_instructions(&self) -> Vec<Insn> { + // Emit instructions with labels, expanding branch parameters + let mut insns = Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY); + + let block_ids = self.block_order(); + let num_blocks = block_ids.len(); + + for (i, block_id) in block_ids.iter().enumerate() { + let block = &self.basic_blocks[block_id.0]; + // Entry blocks shouldn't ever be preceded by something that can + // stomp on this block. + if !block.is_entry { + insns.push(Insn::PadPatchPoint); + } + + // Process each instruction, expanding branch params if needed + for insn in &block.insns { + self.expand_branch_insn(insn, &mut insns); + } + + // Eliminate redundant jumps: if the last instruction is an + // unconditional jump to the next block in the linear order, + // remove it and let execution fall through. + if let Some(next_block_id) = block_ids.get(i + 1) { + let next_label = self.block_label(*next_block_id); + if let Some(Insn::Jmp(Target::Label(label))) = insns.last() { + if *label == next_label { + insns.pop(); + } + } + } + + // Make sure we don't stomp on the next function + if block_id.0 == num_blocks - 1 { + insns.push(Insn::PadPatchPoint); + } } + insns } - /// Build an Opnd::VReg and initialize its LiveRange - pub(super) fn new_vreg(&mut self, num_bits: u8) -> Opnd { - let vreg = Opnd::VReg { idx: self.live_ranges.len(), num_bits }; - self.live_ranges.push(LiveRange { start: None, end: None }); + /// Expand and linearize a branch instruction: + /// 1. If the branch has Target::Block with arguments, insert a ParallelMov first + /// 2. Convert Target::Block to Target::Label + /// 3. Push the converted instruction + fn expand_branch_insn(&self, insn: &Insn, insns: &mut Vec<Insn>) { + // Helper to process branch arguments and return the label target + let process_edge = |edge: &BranchEdge| -> Label { + self.block_label(edge.target) + }; + + // Convert Target::Block to Target::Label, processing args if needed + let stripped_insn = match insn { + Insn::Jmp(Target::Block(edge)) => Insn::Jmp(Target::Label(process_edge(edge))), + Insn::Jz(Target::Block(edge)) => Insn::Jz(Target::Label(process_edge(edge))), + Insn::Jnz(Target::Block(edge)) => Insn::Jnz(Target::Label(process_edge(edge))), + Insn::Je(Target::Block(edge)) => Insn::Je(Target::Label(process_edge(edge))), + Insn::Jne(Target::Block(edge)) => Insn::Jne(Target::Label(process_edge(edge))), + Insn::Jl(Target::Block(edge)) => Insn::Jl(Target::Label(process_edge(edge))), + Insn::Jg(Target::Block(edge)) => Insn::Jg(Target::Label(process_edge(edge))), + Insn::Jge(Target::Block(edge)) => Insn::Jge(Target::Label(process_edge(edge))), + Insn::Jbe(Target::Block(edge)) => Insn::Jbe(Target::Label(process_edge(edge))), + Insn::Jb(Target::Block(edge)) => Insn::Jb(Target::Label(process_edge(edge))), + Insn::Jo(Target::Block(edge)) => Insn::Jo(Target::Label(process_edge(edge))), + Insn::JoMul(Target::Block(edge)) => Insn::JoMul(Target::Label(process_edge(edge))), + Insn::Joz(opnd, Target::Block(edge)) => Insn::Joz(*opnd, Target::Label(process_edge(edge))), + Insn::Jonz(opnd, Target::Block(edge)) => Insn::Jonz(*opnd, Target::Label(process_edge(edge))), + _ => insn.clone() + }; + + // Push the stripped instruction + insns.push(stripped_insn); + } + + // Get the label for a given block by extracting it from the first instruction. + pub(super) fn block_label(&self, block_id: BlockId) -> Label { + let block = &self.basic_blocks[block_id.0]; + match block.insns.first() { + Some(Insn::Label(Target::Label(label))) => *label, + other => panic!("Expected first instruction of block {:?} to be a Label, but found: {:?}", block_id, other), + } + } + + pub fn expect_leaf_ccall(&mut self, stack_size: usize) { + self.leaf_ccall_stack_size = Some(stack_size); + } + + fn set_stack_canary(&mut self) -> Option<Opnd> { + if cfg!(feature = "runtime_checks") { + if let Some(stack_size) = self.leaf_ccall_stack_size.take() { + let canary_addr = self.lea(Opnd::mem(64, SP, (stack_size as i32) * SIZEOF_VALUE_I32)); + let canary_opnd = Opnd::mem(64, canary_addr, 0); + self.mov(canary_opnd, vm_stack_canary().into()); + return Some(canary_opnd) + } + } + None + } + + fn clear_stack_canary(&mut self, canary_opnd: Option<Opnd>){ + if let Some(canary_opnd) = canary_opnd { + self.store(canary_opnd, 0.into()); + }; + } + + /// Build an Opnd::VReg + pub fn new_vreg(&mut self, num_bits: u8) -> Opnd { + let vreg = Opnd::VReg { idx: VRegId(self.num_vregs), num_bits }; + self.num_vregs += 1; vreg } + /// Build an Opnd::VReg for use as a block parameter. + pub fn new_block_param(&mut self, num_bits: u8) -> Opnd { + self.new_vreg(num_bits) + } + /// Append an instruction onto the current list of instructions and update /// the live ranges of any instructions whose outputs are being used as /// operands to this instruction. pub fn push_insn(&mut self, insn: Insn) { - // Index of this instruction - let insn_idx = self.insns.len(); - - // Initialize the live range of the output VReg to insn_idx..=insn_idx - if let Some(Opnd::VReg { idx, .. }) = insn.out_opnd() { - assert!(*idx < self.live_ranges.len()); - assert_eq!(self.live_ranges[*idx], LiveRange { start: None, end: None }); - self.live_ranges[*idx] = LiveRange { start: Some(insn_idx), end: Some(insn_idx) }; - } - - // If we find any VReg from previous instructions, extend the live range to insn_idx - let mut opnd_iter = insn.opnd_iter(); - while let Some(opnd) = opnd_iter.next() { - match *opnd { - Opnd::VReg { idx, .. } | - Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { - assert!(idx < self.live_ranges.len()); - assert_ne!(self.live_ranges[idx].end, None); - self.live_ranges[idx].end = Some(self.live_ranges[idx].end().max(insn_idx)); - } - _ => {} - } + // If this Assembler should not accept scratch registers, assert no use of them. + if !self.accept_scratch_reg { + insn.for_each_operand(|opnd| { + assert!(!Self::has_scratch_reg(opnd), "should not use scratch register: {opnd:?}"); + }); } - self.insns.push(insn); + self.idx += 1; + + self.current_block().push_insn(insn); } /// Create a new label instance that we can jump to @@ -1240,19 +1704,25 @@ impl Assembler Target::Label(label) } - // Shuffle register moves, sometimes adding extra moves using SCRATCH_REG, + // Shuffle register moves, sometimes adding extra moves using scratch_reg, // so that they will not rewrite each other before they are used. - pub fn resolve_parallel_moves(old_moves: &Vec<(Reg, Opnd)>) -> Vec<(Reg, Opnd)> { + pub fn resolve_parallel_moves(old_moves: &[(Opnd, Opnd)], scratch_opnd: Option<Opnd>) -> Option<Vec<(Opnd, Opnd)>> { // Return the index of a move whose destination is not used as a source if any. - fn find_safe_move(moves: &Vec<(Reg, Opnd)>) -> Option<usize> { - moves.iter().enumerate().find(|&(_, &(dest_reg, _))| { - moves.iter().all(|&(_, src_opnd)| src_opnd != Opnd::Reg(dest_reg)) + fn find_safe_move(moves: &[(Opnd, Opnd)]) -> Option<usize> { + moves.iter().enumerate().find(|&(_, &(dst, src))| { + // Check if `dst` is used in other moves. If `dst` is not used elsewhere, it's safe to write into `dst` now. + moves.iter().filter(|&&other_move| other_move != (dst, src)).all(|&(other_dst, other_src)| + match dst { + Opnd::Reg(reg) => !Assembler::has_reg(other_dst, reg) && !Assembler::has_reg(other_src, reg), + _ => other_dst != dst && other_src != dst, + } + ) }).map(|(index, _)| index) } // Remove moves whose source and destination are the same - let mut old_moves: Vec<(Reg, Opnd)> = old_moves.clone().into_iter() - .filter(|&(reg, opnd)| Opnd::Reg(reg) != opnd).collect(); + let mut old_moves: Vec<(Opnd, Opnd)> = old_moves.iter().copied() + .filter(|&(dst, src)| dst != src).collect(); let mut new_moves = vec![]; while !old_moves.is_empty() { @@ -1261,283 +1731,618 @@ impl Assembler new_moves.push(old_moves.remove(index)); } - // No safe move. Load the source of one move into SCRATCH_REG, and - // then load SCRATCH_REG into the destination when it's safe. + // No safe move. Load the source of one move into scratch_opnd, and + // then load scratch_opnd into the destination when it's safe. if !old_moves.is_empty() { - // Make sure it's safe to use SCRATCH_REG - assert!(old_moves.iter().all(|&(_, opnd)| opnd != Opnd::Reg(Assembler::SCRATCH_REG))); - - // Move SCRATCH <- opnd, and delay reg <- SCRATCH - let (reg, opnd) = old_moves.remove(0); - new_moves.push((Assembler::SCRATCH_REG, opnd)); - old_moves.push((reg, Opnd::Reg(Assembler::SCRATCH_REG))); - } - } - new_moves - } - - /// Sets the out field on the various instructions that require allocated - /// registers because their output is used as the operand on a subsequent - /// instruction. This is our implementation of the linear scan algorithm. - pub(super) fn alloc_regs(mut self, regs: Vec<Reg>) -> Option<Assembler> { - // Dump live registers for register spill debugging. - fn dump_live_regs(insns: Vec<Insn>, live_ranges: Vec<LiveRange>, num_regs: usize, spill_index: usize) { - // Convert live_ranges to live_regs: the number of live registers at each index - let mut live_regs: Vec<usize> = vec![]; - for insn_idx in 0..insns.len() { - let live_count = live_ranges.iter().filter(|range| - match (range.start, range.end) { - (Some(start), Some(end)) => start <= insn_idx && insn_idx <= end, - _ => false, - } - ).count(); - live_regs.push(live_count); + // If scratch_opnd is None, return None and leave it to *_split_with_scratch_regs to resolve it. + let scratch_opnd = scratch_opnd?; + let scratch_reg = scratch_opnd.unwrap_reg(); + // Make sure it's safe to use scratch_reg + assert!(old_moves.iter().all(|&(dst, src)| !Self::has_reg(dst, scratch_reg) && !Self::has_reg(src, scratch_reg))); + + // Move scratch_opnd <- src, and delay dst <- scratch_opnd + let (dst, src) = old_moves.remove(0); + new_moves.push((scratch_opnd, src)); + old_moves.push((dst, scratch_opnd)); } + } + Some(new_moves) + } - // Dump insns along with live registers - for (insn_idx, insn) in insns.iter().enumerate() { - eprint!("{:3} ", if spill_index == insn_idx { "==>" } else { "" }); - for reg in 0..=num_regs { - eprint!("{:1}", if reg < live_regs[insn_idx] { "|" } else { "" }); + /// Discover vregs that should preferentially reuse a physical register, + /// such as a newborn vreg immediately moved into a preg in the next instruction. + pub fn preferred_register_assignments(&self, intervals: &[Interval]) -> Vec<Option<Reg>> { + let mut preferred = vec![None; self.num_vregs]; + + for block in &self.basic_blocks { + let mut prev_insn: Option<(InsnId, &Insn)> = None; + + for (insn, insn_id) in block.insns.iter().zip(block.insn_ids.iter()) { + let Some(insn_id) = insn_id else { continue; }; + + if !matches!(insn, Insn::Label(_)) { + if let ( + Some((prev_id, prev)), + Insn::Mov { + dest: Opnd::Reg(dest_reg), + src: Opnd::VReg { idx, .. }, + }, + ) = (prev_insn, insn) + { + if let Some(Opnd::VReg { idx: out_idx, .. }) = prev.out_opnd() { + if out_idx == idx + && intervals[idx.0].born_at(prev_id.0) + && intervals[idx.0].dies_at(insn_id.0) + { + preferred[idx.0].get_or_insert(*dest_reg); + } + } + } + + prev_insn = Some((*insn_id, insn)); } - eprintln!(" [{:3}] {:?}", insn_idx, insn); } } - // First, create the pool of registers. - let mut pool = RegisterPool::new(regs.clone()); - - // Mapping between VReg and allocated VReg for each VReg index. - // None if no register has been allocated for the VReg. - let mut reg_mapping: Vec<Option<Reg>> = vec![None; self.live_ranges.len()]; - - // List of registers saved before a C call, paired with the VReg index. - let mut saved_regs: Vec<(Reg, usize)> = vec![]; - - // live_ranges is indexed by original `index` given by the iterator. - let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); - let mut iterator = self.insns.into_iter().enumerate().peekable(); - let mut asm = Assembler::new_with_label_names(take(&mut self.label_names), live_ranges.len()); - - while let Some((index, mut insn)) = iterator.next() { - let before_ccall = match (&insn, iterator.peek().map(|(_, insn)| insn)) { - (Insn::ParallelMov { .. }, Some(Insn::CCall { .. })) | - (Insn::CCall { .. }, _) if !pool.is_empty() => { - // If C_RET_REG is in use, move it to another register. - // This must happen before last-use registers are deallocated. - if let Some(vreg_idx) = pool.vreg_for(&C_RET_REG) { - let new_reg = if let Some(new_reg) = pool.alloc_reg(vreg_idx) { - new_reg - } else { - debug!("spilling VReg is not implemented yet, can't evacuate C_RET_REG on CCall"); - return None; - }; - asm.mov(Opnd::Reg(new_reg), C_RET_OPND); - pool.dealloc_reg(&C_RET_REG); - reg_mapping[vreg_idx] = Some(new_reg); - } + preferred + } + // TODO: We want to make the following refactoring so that we DON'T have + // to parcopy in to entry blocks + // + // * Move Allocation to Interval + // * Pre-allocate pinned regs + // * Update linear scan to handle pinned LRs + // + pub fn linear_scan( + &self, + intervals: Vec<Interval>, + num_registers: usize, + preferred_registers: &[Option<Reg>], + ) -> (Vec<Option<Allocation>>, usize) { + assert_eq!(preferred_registers.len(), intervals.len()); + + let mut free_registers: BTreeSet<usize> = (0..num_registers).collect(); + let mut active: Vec<&Interval> = Vec::new(); // vreg indices sorted by increasing end point + let mut assignment: Vec<Option<Allocation>> = vec![None; intervals.len()]; + let mut num_stack_slots: usize = 0; + + // Collect vreg indices that have valid ranges, sorted by start point + let mut sorted_intervals: Vec<Interval> = intervals.iter() + .filter(|i| i.range.start.is_some() && i.range.end.is_some()) + .cloned() + .collect(); + sorted_intervals.sort_by_key(|i| i.range.start.unwrap()); + + for interval in &sorted_intervals { + // Expire old intervals + active.retain(|&active_interval| { + if active_interval.range.end.unwrap() > interval.range.start.unwrap() { true - }, - _ => false, - }; - - // Check if this is the last instruction that uses an operand that - // spans more than one instruction. In that case, return the - // allocated register to the pool. - for opnd in insn.opnd_iter() { - match *opnd { - Opnd::VReg { idx, .. } | - Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { - // We're going to check if this is the last instruction that - // uses this operand. If it is, we can return the allocated - // register to the pool. - if live_ranges[idx].end() == index { - if let Some(reg) = reg_mapping[idx] { - pool.dealloc_reg(®); - } else { - unreachable!("no register allocated for insn {:?}", insn); - } + } else { + if let Some(allocation) = assignment[active_interval.id] { + if let Some(reg) = allocation.alloc_pool_index(num_registers) { + assert!( + free_registers.insert(reg), + "attempted to return allocator register {:?} to the free pool more than once", + allocation.assigned_reg().unwrap(), + ); + } else { + assert!( + allocation.assigned_reg().is_none_or(|reg| { + crate::backend::current::ALLOC_REGS + .iter() + .take(num_registers) + .all(|candidate| candidate.reg_no != reg.reg_no) + }), + "attempted to return non-allocatable register {:?} to the allocator pool", + allocation.assigned_reg().unwrap(), + ); } } - _ => {} + false + } + }); + + let preferred_reg = preferred_registers[interval.id]; + let preferred_taken = preferred_reg.is_some_and(|reg| { + active.iter().any(|active_interval| { + assignment[active_interval.id] + .and_then(|alloc| alloc.assigned_reg()) + .is_some_and(|active_reg| active_reg.reg_no == reg.reg_no) + }) + }); + + if let Some(preferred_reg) = preferred_reg.filter(|_| !preferred_taken) { + if let Some(reg_idx) = Allocation::Fixed(preferred_reg).alloc_pool_index(num_registers) { + if free_registers.remove(®_idx) { + assignment[interval.id] = Some(Allocation::Fixed(preferred_reg)); + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); + continue; + } + } else { + assignment[interval.id] = Some(Allocation::Fixed(preferred_reg)); + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); + continue; } } - // Save caller-saved registers on a C call. - if before_ccall { - // Find all live registers - saved_regs = pool.live_regs(); - - // Save live registers - for &(reg, _) in saved_regs.iter() { - asm.cpush(Opnd::Reg(reg)); - pool.dealloc_reg(®); - } - // On x86_64, maintain 16-byte stack alignment - if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { - asm.cpush(Opnd::Reg(saved_regs.last().unwrap().0)); + if free_registers.is_empty() { + // Spill: pick the longest-lived active interval (last in sorted active) + // but only from the allocatable register pool. Fixed register + // assignments represent preferred/pinned physical registers + // (for example SP) and should not be selected as spill victims. + let spill = active.iter().rev().copied().find(|active_interval| { + matches!(assignment[active_interval.id], Some(Allocation::Reg(_))) + }); + let slot = Allocation::Stack(num_stack_slots); + num_stack_slots += 1; + + if let Some(spill) = spill.filter(|spill| spill.range.end.unwrap() > interval.range.end.unwrap()) { + // Spill the last active interval; give its register to current + assignment[interval.id] = assignment[spill.id]; + assignment[spill.id] = Some(slot); + let spill_idx = active.iter().position(|active_interval| active_interval.id == spill.id).unwrap(); + active.remove(spill_idx); + // Insert current into sorted active + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); + } else { + // Spill the current interval + assignment[interval.id] = Some(slot); } + } else { + // Allocate lowest free register + let reg = *free_registers.iter().min().unwrap(); + free_registers.remove(®); + assignment[interval.id] = Some(Allocation::Reg(reg)); + // Insert into sorted active + let insert_idx = active.partition_point(|&i| i.range.end.unwrap() < interval.range.end.unwrap()); + active.insert(insert_idx, &interval); } + } - // Allocate a register for the output operand if it exists - let vreg_idx = match insn.out_opnd() { - Some(Opnd::VReg { idx, .. }) => Some(*idx), - _ => None, - }; - if vreg_idx.is_some() { - if live_ranges[vreg_idx.unwrap()].end() == index { - debug!("Allocating a register for VReg({}) at instruction index {} even though it does not live past this index", vreg_idx.unwrap(), index); - } - // This is going to be the output operand that we will set on the - // instruction. CCall and LiveReg need to use a specific register. - let mut out_reg = match insn { - Insn::CCall { .. } => { - Some(pool.take_reg(&C_RET_REG, vreg_idx.unwrap())) - } - Insn::LiveReg { opnd, .. } => { - let reg = opnd.unwrap_reg(); - Some(pool.take_reg(®, vreg_idx.unwrap())) - } - _ => None - }; + (assignment, num_stack_slots) + } - // If this instruction's first operand maps to a register and - // this is the last use of the register, reuse the register - // We do this to improve register allocation on x86 - // e.g. out = add(reg0, reg1) - // reg0 = add(reg0, reg1) - if out_reg.is_none() { - let mut opnd_iter = insn.opnd_iter(); - - if let Some(Opnd::VReg{ idx, .. }) = opnd_iter.next() { - if live_ranges[*idx].end() == index { - if let Some(reg) = reg_mapping[*idx] { - out_reg = Some(pool.take_reg(®, vreg_idx.unwrap())); - } - } - } + /// Resolve SSA block parameters by inserting sequentialized move instructions + /// at block boundaries. This is SSA deconstruction: after linear_scan assigns + /// registers/stack slots, we lower block parameter passing to explicit moves. + pub fn resolve_ssa(&mut self, _intervals: &[Interval], assignments: &[Option<Allocation>]) { + use crate::backend::parcopy; + use crate::backend::current::SCRATCH_REG; + + // Count predecessors for each block + let mut num_predecessors: HashMap<BlockId, usize> = HashMap::new(); + for block_id in self.block_order() { + for succ in self.basic_blocks[block_id.0].successors() { + *num_predecessors.entry(succ).or_insert(0) += 1; + } + } + + // Collect block order upfront so we don't borrow self while mutating + let block_order = self.block_order(); + + // This code is iterating over each block in our CFG and inserting + // copy instructions at each edge. + for &pred_id in &block_order { + let pred_hir_block_id = self.basic_blocks[pred_id.0].hir_block_id; + let pred_rpo_index = self.basic_blocks[pred_id.0].rpo_index; + let EdgePair(edge1, edge2) = self.basic_blocks[pred_id.0].edges(); + + let edges: Vec<BranchEdge> = [edge1, edge2].into_iter().flatten().collect(); + let num_successors = edges.len(); + + for edge in edges { + let successor = edge.target; + let params = self.basic_blocks[successor.0].parameters.clone(); + + // Build the list of register-to-register copies and immediate moves. + // Rewrite VRegs to physical registers BEFORE sequentialization so + // the parcopy algorithm can see real physical register conflicts. + let reg_copies: Vec<parcopy::RegisterCopy<Opnd>> = edge.args + .iter() + .zip(params.iter()) + .filter(|(_arg, param)| assignments[param.vreg_idx().0].is_some() ) + .map(|(arg, param)| parcopy::RegisterCopy::<Opnd> { + destination: Self::rewritten_opnd(*param, assignments), + source: Self::rewritten_opnd(*arg, assignments), + }) + .filter(|copy| copy.source != copy.destination) + .collect(); + + // Sequentialize register copies. + // Copies must use physical registers, not VRegs, so the + // parcopy algorithm can detect physical register conflicts. + debug_assert!(reg_copies.iter().all(|c| !c.source.is_vreg() && !c.destination.is_vreg()), + "parcopy must operate on physical registers, not VRegs"); + let sequentialized = parcopy::sequentialize_register(®_copies, Opnd::Reg(SCRATCH_REG)); + let moves: Vec<Insn> = sequentialized + .iter() + .map(|copy| match copy.source { + Opnd::Value(_) => Insn::LoadInto { dest: copy.destination, opnd: copy.source }, + _ => Insn::Mov { dest: copy.destination, src: copy.source }, + }) + .collect(); + + if moves.is_empty() { + continue; } - // Allocate a new register for this instruction if one is not - // already allocated. - if out_reg.is_none() { - out_reg = match &insn { - _ => match pool.alloc_reg(vreg_idx.unwrap()) { - Some(reg) => Some(reg), - None => { - if get_option!(debug) { - let mut insns = asm.insns; - insns.push(insn); - while let Some((_, insn)) = iterator.next() { - insns.push(insn); - } - dump_live_regs(insns, live_ranges, regs.len(), index); + let num_preds = *num_predecessors.get(&successor).unwrap_or(&0); + if num_preds > 1 && num_successors > 1 { + // Critical edge: create interstitial block + let new_block_id = self.new_block(pred_hir_block_id, false, pred_rpo_index); + let label = self.new_label("split"); + self.basic_blocks[new_block_id.0].push_insn(Insn::Label(label)); + for mov in moves { + self.basic_blocks[new_block_id.0].push_insn(mov); + } + self.basic_blocks[new_block_id.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: successor, + args: vec![], + }))); + + // Redirect predecessor's branch to the new block + let pred_insns = &mut self.basic_blocks[pred_id.0].insns; + for insn in pred_insns.iter_mut() { + if let Some(target) = insn.target_mut() { + if let Target::Block(e) = target { + if e.target == successor { + e.target = new_block_id; + e.args = vec![]; + break; } - debug!("Register spill not supported"); - return None; } } - }; + } + } else if num_successors > 1 { + // Multi-succ: insert at start of successor (after Label) + for (i, mov) in moves.into_iter().enumerate() { + self.basic_blocks[successor.0].insns.insert(1 + i, mov); + self.basic_blocks[successor.0].insn_ids.insert(1 + i, None); + } + } else { + assert_eq!(num_successors, 1); + // Single-succ: insert at end of predecessor before terminator + let len = self.basic_blocks[pred_id.0].insns.len(); + for (i, mov) in moves.into_iter().enumerate() { + self.basic_blocks[pred_id.0].insns.insert(len - 1 + i, mov); + self.basic_blocks[pred_id.0].insn_ids.insert(len - 1 + i, None); + } } + } + } - // Set the output operand on the instruction - let out_num_bits = Opnd::match_num_bits_iter(insn.opnd_iter()); - - // If we have gotten to this point, then we're sure we have an - // output operand on this instruction because the live range - // extends beyond the index of the instruction. - let out = insn.out_opnd_mut().unwrap(); - let reg = out_reg.unwrap().with_num_bits(out_num_bits); - reg_mapping[out.vreg_idx()] = Some(reg); - *out = Opnd::Reg(reg); + // Handle entry block parameters: move from calling-convention registers + // to their allocated locations, just like inter-block edge moves above. + for &block_id in &block_order { + if !self.basic_blocks[block_id.0].is_entry { continue; } + if self.basic_blocks[block_id.0].is_dummy() { continue; } + let params = self.basic_blocks[block_id.0].parameters.clone(); + + // JIT-to-JIT entries that would need more argument registers should + // be unreachable because can_direct_send() refuses to call them. + // Keep compiling the function body, but make the unsupported entry + // abort if control ever reaches it. TODO: Remove this (Shopify/ruby#916) + if params.len() > C_ARG_OPNDS.len() { + let insert_pos = self.basic_blocks[block_id.0].insns.iter() + .position(|insn| matches!(insn, Insn::FrameSetup { .. })) + .or_else(|| self.basic_blocks[block_id.0].insns.iter().position(|insn| matches!(insn, Insn::Label(_))).map(|idx| idx + 1)) + .unwrap_or(0); + self.basic_blocks[block_id.0].insns.insert(insert_pos, Insn::Abort); + self.basic_blocks[block_id.0].insn_ids.insert(insert_pos, None); + continue; } - // Replace VReg and Param operands by their corresponding register - let mut opnd_iter = insn.opnd_iter_mut(); - while let Some(opnd) = opnd_iter.next() { - match *opnd { - Opnd::VReg { idx, num_bits } => { - *opnd = Opnd::Reg(reg_mapping[idx].unwrap()).with_num_bits(num_bits); + // Rewrite VRegs to physical registers before sequentialization + // so the parcopy algorithm can detect physical register conflicts. + let reg_copies: Vec<parcopy::RegisterCopy<Opnd>> = params.iter().enumerate() + .map(|(i, param)| parcopy::RegisterCopy::<Opnd> { + source: C_ARG_OPNDS[i], + destination: Self::rewritten_opnd(*param, assignments), + }) + .filter(|copy| copy.source != copy.destination) + .collect(); + + debug_assert!(reg_copies.iter().all(|c| !c.source.is_vreg() && !c.destination.is_vreg()), + "parcopy must operate on physical registers, not VRegs"); + let sequentialized = parcopy::sequentialize_register(®_copies, Opnd::Reg(SCRATCH_REG)); + let moves: Vec<Insn> = sequentialized + .iter() + .map(|copy| match copy.source { + Opnd::Value(_) => Insn::LoadInto { + dest: copy.destination, + opnd: copy.source, }, - Opnd::Mem(Mem { base: MemBase::VReg(idx), disp, num_bits }) => { - let base = MemBase::Reg(reg_mapping[idx].unwrap().reg_no); - *opnd = Opnd::Mem(Mem { base, disp, num_bits }); - } - _ => {}, - } + _ => Insn::Mov { + dest: copy.destination, + src: copy.source, + }, + }) + .collect(); + + // Find the position after FrameSetup to insert moves + let insert_pos = self.basic_blocks[block_id.0].insns.iter() + .position(|insn| matches!(insn, Insn::FrameSetup { .. })) + .or_else(|| self.basic_blocks[block_id.0].insns.iter().position(|insn| matches!(insn, Insn::Label(_))).map(|idx| idx + 1)) + .unwrap_or(0); + + for (i, mov) in moves.into_iter().enumerate() { + self.basic_blocks[block_id.0].insns.insert(insert_pos + i, mov); + self.basic_blocks[block_id.0].insn_ids.insert(insert_pos + i, None); } + } - // If we have an output that dies at its definition (it is unused), free up the - // register - if let Some(idx) = vreg_idx { - if live_ranges[idx].end() == index { - if let Some(reg) = reg_mapping[idx] { - pool.dealloc_reg(®); - } else { - unreachable!("no register allocated for insn {:?}", insn); - } + // Clear edge args on all branch instructions since the moves have been + // materialized as explicit Mov instructions. This prevents + // linearize_instructions from generating redundant ParallelMov instructions. + for block_id in &block_order { + for insn in &mut self.basic_blocks[block_id.0].insns { + if let Some(Target::Block(edge)) = insn.target_mut() { + edge.args.clear(); } } + } - // Push instruction(s) - let is_ccall = matches!(insn, Insn::CCall { .. }); - match insn { - Insn::ParallelMov { moves } => { - // Now that register allocation is done, it's ready to resolve parallel moves. - for (reg, opnd) in Self::resolve_parallel_moves(&moves) { - asm.load_into(Opnd::Reg(reg), opnd); + self.rewrite_instructions(assignments); + } + + /// Handle caller-saved registers around CCall instructions. + /// For each CCall, push live caller-saved registers, set up arguments + /// in C calling convention registers, and pop saved registers after. + pub fn handle_caller_saved_regs( + &mut self, + intervals: &[Interval], + assignments: &[Option<Allocation>], + regs: &[Reg], + ) { + use crate::backend::parcopy; + use crate::backend::current::{C_RET_OPND, SCRATCH_REG, ALLOC_REGS}; + + for block_id in self.block_order() { + let block = &mut self.basic_blocks[block_id.0]; + let old_insns = take(&mut block.insns); + let old_ids = take(&mut block.insn_ids); + + let mut new_insns = Vec::with_capacity(old_insns.len()); + let mut new_ids = Vec::with_capacity(old_ids.len()); + + for (insn, insn_id) in old_insns.into_iter().zip(old_ids.into_iter()) { + if let Insn::CCall { opnds, out, start_marker, end_marker, fptr } = insn { + let insn_number = insn_id.map(|id| id.0).unwrap_or(0); + // Do we have a case where a ccall is emitted, but nobody + // uses the result? + let call_result_live = out.is_vreg() + && intervals[out.vreg_idx().0] + .range + .end + .is_some_and(|end| end > insn_number); + + // Find survivors: intervals that survive this Call instruction + // We need to preserve the "surviving" registers past the ccall, + // so we're going to push them all on the stack, then pop + // after we make the ccall + let survivors: Vec<usize> = intervals.iter() + .filter(|interval| { + interval.has_bounds() + && interval.survives(insn_number) + && assignments[interval.id].and_then(|alloc| alloc.alloc_pool_index(ALLOC_REGS.len())).is_some() + }) + .map(|interval| interval.id) + .collect(); + + let survivor_regs: Vec<Opnd> = survivors.iter() + .map(|&s| match assignments[s].unwrap() { + Allocation::Reg(n) => Opnd::Reg(ALLOC_REGS[n]), + Allocation::Fixed(reg) => Opnd::Reg(reg), + _ => unreachable!(), + }) + .collect(); + + // Push all survivors on the stack, pairing adjacent pushes when possible. + for group in survivor_regs.chunks(2) { + match group { + &[left, right] => new_insns.push(Insn::CPushPair(left, right)), + &[reg] => new_insns.push(Insn::CPushPair(reg, 0.into())), + _ => unreachable!(), + } + new_ids.push(None); } - } - Insn::CCall { opnds, fptr, start_marker, end_marker, out } => { - // Split start_marker and end_marker here to avoid inserting push/pop between them. - if let Some(start_marker) = start_marker { - asm.push_insn(Insn::PosMarker(start_marker)); + // Extract arguments from CCall, clear opnds + + assert!(opnds.len() <= regs.len()); + + // Sequentialize argument moves: each arg goes to regs[i] + let reg_copies: Vec<parcopy::RegisterCopy<Opnd>> = opnds + .iter() + .zip(regs.iter()) + .map(|(arg, param)| parcopy::RegisterCopy::<Opnd> { + destination: Opnd::Reg(*param), + source: Self::rewritten_opnd(*arg, assignments), + }) + .filter(|copy| copy.source != copy.destination) + .collect(); + + debug_assert!(reg_copies.iter().all(|c| !c.source.is_vreg() && !c.destination.is_vreg()), + "parcopy must operate on physical registers, not VRegs"); + let sequentialized = parcopy::sequentialize_register(®_copies, Opnd::Reg(SCRATCH_REG)); + + for copy in sequentialized { + new_insns.push(match copy.source { + Opnd::Value(_) => Insn::LoadInto { dest: copy.destination, opnd: copy.source }, + _ => Insn::Mov { dest: copy.destination, src: copy.source }, + }); + new_ids.push(None); } - asm.push_insn(Insn::CCall { opnds, fptr, start_marker: None, end_marker: None, out }); - if let Some(end_marker) = end_marker { - asm.push_insn(Insn::PosMarker(end_marker)); + + // Extract PosMarkers from the CCall so they get emitted + // as separate instructions at the right code positions. + // Emit start_marker PosMarker before the CCall + if let Some(marker) = start_marker { + new_insns.push(Insn::PosMarker(marker)); + new_ids.push(None); } + + // The CCall itself + new_insns.push(Insn::CCall { + out: C_RET_OPND, + opnds: vec![], // We've moved everything in to ccall regs, so this should + // be empty now + start_marker: None, + end_marker: None, + fptr + }); + new_ids.push(insn_id); + + // Emit end_marker PosMarker after the CCall + if let Some(marker) = end_marker { + new_insns.push(Insn::PosMarker(marker)); + new_ids.push(None); + } + + if survivors.is_empty() { + if call_result_live { + // No survivors to restore -- move result directly to output. + let out = Self::rewritten_opnd(out, assignments); + new_insns.push(Insn::Mov { dest: out, src: C_RET_OPND }); + new_ids.push(None); + } + } else { + if call_result_live { + // Save CCall result to scratch immediately, before pops + // can clobber either C_RET or the output register. + new_insns.push(Insn::Mov { dest: Opnd::Reg(SCRATCH_REG), src: C_RET_OPND }); + new_ids.push(None); + } + + // Restore all survivors in reverse stack order, pairing adjacent pops when possible. + for group in survivor_regs.chunks(2).rev() { + match group { + &[reg] => new_insns.push(Insn::CPopPairInto(reg, reg)), + &[left, right] => new_insns.push(Insn::CPopPairInto(right, left)), + _ => unreachable!(), + } + new_ids.push(None); + } + + if call_result_live { + // Move result from scratch to output AFTER all pops. + let out = Self::rewritten_opnd(out, assignments); + new_insns.push(Insn::Mov { dest: out, src: Opnd::Reg(SCRATCH_REG) }); + new_ids.push(None); + } + } + } else { + new_insns.push(insn); + new_ids.push(insn_id); } - Insn::Mov { src, dest } | Insn::LoadInto { dest, opnd: src } if src == dest => { - // Remove no-op move now that VReg are resolved to physical Reg + } + + let block = &mut self.basic_blocks[block_id.0]; + block.insns = new_insns; + block.insn_ids = new_ids; + } + } + + /// Walk every instruction and replace VReg operands with the physical + /// register (or stack slot) from the allocation assignments. + fn rewrite_instructions(&mut self, assignments: &[Option<Allocation>]) { + for block_id in self.block_order() { + for insn in self.basic_blocks[block_id.0].insns.iter_mut() { + insn.for_each_operand_mut(|opnd| { + Self::rewrite_opnd(opnd, assignments); + }); + if let Some(out) = insn.out_opnd_mut() { + Self::rewrite_opnd(out, assignments); } - _ => asm.push_insn(insn), } + } + } - // After a C call, restore caller-saved registers - if is_ccall { - // On x86_64, maintain 16-byte stack alignment - if cfg!(target_arch = "x86_64") && saved_regs.len() % 2 == 1 { - asm.cpop_into(Opnd::Reg(saved_regs.last().unwrap().0.clone())); + fn rewritten_opnd(mut opnd: Opnd, assignments: &[Option<Allocation>]) -> Opnd { + Self::rewrite_opnd(&mut opnd, assignments); + opnd + } + + fn rewrite_opnd(opnd: &mut Opnd, assignments: &[Option<Allocation>]) { + use crate::backend::current::ALLOC_REGS; + let regs = &ALLOC_REGS; + + match opnd { + Opnd::VReg { idx, num_bits } => { + if let Some(assignment) = assignments[idx.0] { + match assignment { + Allocation::Reg(n) => { + let mut reg = regs[n]; + reg.num_bits = *num_bits; + *opnd = Opnd::Reg(reg); + } + Allocation::Fixed(mut reg) => { + reg.num_bits = *num_bits; + *opnd = Opnd::Reg(reg); + } + Allocation::Stack(n) => { + let num_bits = *num_bits; + *opnd = Opnd::Mem(Mem { + base: MemBase::Stack { stack_idx: n, num_bits }, + disp: 0, + num_bits, + }); + } + } + } else { + panic!("Expected assignment for {opnd}"); } - // Restore saved registers - for &(reg, vreg_idx) in saved_regs.iter().rev() { - asm.cpop_into(Opnd::Reg(reg)); - pool.take_reg(®, vreg_idx); + } + Opnd::Mem(Mem { base: MemBase::VReg(idx), .. }) => { + match assignments[idx.0].unwrap() { + Allocation::Reg(n) => { + if let Opnd::Mem(mem) = opnd { + mem.base = MemBase::Reg(regs[n].reg_no); + } + } + Allocation::Fixed(reg) => { + if let Opnd::Mem(mem) = opnd { + mem.base = MemBase::Reg(reg.reg_no); + } + } + Allocation::Stack(n) => { + // The VReg used as a memory base was spilled to a stack slot. + // Mark it as StackIndirect so arm64_scratch_split can load + // the pointer from the stack into a scratch register. + if let Opnd::Mem(mem) = opnd { + mem.base = MemBase::StackIndirect { stack_idx: n }; + } + } } - saved_regs.clear(); } + _ => {} } - - assert!(pool.is_empty(), "Expected all registers to be returned to the pool"); - Some(asm) } /// Compile the instructions down to machine code. /// Can fail due to lack of code memory and inopportune code placement, among other reasons. - #[must_use] - pub fn compile(self, cb: &mut CodeBlock) -> Option<(CodePtr, Vec<CodePtr>)> { + pub fn compile(self, cb: &mut CodeBlock) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { #[cfg(feature = "disasm")] let start_addr = cb.get_write_ptr(); let alloc_regs = Self::get_alloc_regs(); - let ret = self.compile_with_regs(cb, alloc_regs); + let had_dropped_bytes = cb.has_dropped_bytes(); + let ret = self.compile_with_regs(cb, alloc_regs).inspect_err(|err| { + // If we use too much memory to compile the Assembler, it would set cb.dropped_bytes = true. + // To avoid failing future compilation by cb.has_dropped_bytes(), attempt to reset dropped_bytes with + // the current zjit_alloc_bytes() which may be decreased after self is dropped in compile_with_regs(). + if *err == CompileError::OutOfMemory && !had_dropped_bytes { + cb.update_dropped_bytes(); + } + }); #[cfg(feature = "disasm")] - if get_option!(dump_disasm) { + if let Some(dump_disasm) = crate::options::get_option_ref!(dump_disasm).filter(|_| ret.is_ok()) { let end_addr = cb.get_write_ptr(); - let disasm = crate::disasm::disasm_addr_range(cb, start_addr.raw_ptr(cb) as usize, end_addr.raw_ptr(cb) as usize); - println!("{}", disasm); + crate::disasm::dump_disasm_addr_range(cb, start_addr, end_addr, dump_disasm); } ret } @@ -1550,64 +2355,765 @@ impl Assembler self.compile_with_regs(cb, alloc_regs).unwrap() } - /// Compile Target::SideExit and convert it into Target::CodePtr for all instructions - pub fn compile_side_exits(&mut self) { + /// Compile Target::SideExit and convert it into Target::Label for all instructions. + /// Returns the exit code as a list of instructions to be appended after the main + /// code is linearized and split. + pub fn compile_exits(&mut self) -> Vec<Insn> { + /// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit. + fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) { + let SideExit { pc, stack, locals, iseq, .. } = exit; + + // Side exit blocks are not part of the CFG at the moment, + // so we need to manually ensure that patchpoints get padded + // so that nobody stomps on us + asm.pad_patch_point(); + + asm_comment!(asm, "save cfp->pc"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), *pc); + + asm_comment!(asm, "save cfp->sp"); + asm.lea_into(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); + + asm_comment!(asm, "save cfp->iseq"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE::from(*iseq).into()); + + // cfp->block_code and cfp->jit_return are cleared by the materialize_exit trampoline + + if !stack.is_empty() { + asm_comment!(asm, "write stack slots: {}", join_opnds(&stack, ", ")); + for (idx, &opnd) in stack.iter().enumerate() { + asm.store(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), opnd); + } + } + + if !locals.is_empty() { + asm_comment!(asm, "write locals: {}", join_opnds(&locals, ", ")); + for (idx, &opnd) in locals.iter().enumerate() { + asm.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); + } + } + } + + /// Tear down the JIT frame and return to the interpreter. + fn compile_exit_return(asm: &mut Assembler) { + asm_comment!(asm, "exit to the interpreter"); + asm.jmp(Target::CodePtr(ZJITState::get_materialize_exit_trampoline())); + } + + fn compile_exit_recompile(asm: &mut Assembler, exit: &SideExit) { + if let Some(recompile) = &exit.recompile { + let payload = get_or_create_iseq_payload(exit.iseq); + payload.reset_profiles_remaining(recompile.insn_idx as YarvInsnIdx); + use crate::codegen::exit_recompile; + asm_comment!(asm, "profile and maybe recompile"); + asm_ccall!(asm, exit_recompile, + EC, + recompile.iseq, + Opnd::UImm(recompile.insn_idx as u64), + Opnd::Imm(match recompile.strategy { + hir::Recompile::ProfileSend { argc } => argc as i64, + hir::Recompile::ProfileSelf => -1, + }) + ); + } + } + + /// Compile the main side-exit code. The side exit will optionally record a traced exit + /// stack, optionally trigger recompilation, and then return to the interpreter. Shared + /// exits pass no trace reason so they can still be deduplicated by SideExit. + /// IOW, we should never pass a trace reason if we expect the exit to be + /// deduplicated. + fn compile_exit(asm: &mut Assembler, exit: &SideExit, trace_reason: Option<SideExitReason>) { + // Save VM state before the ccall so that + // rb_profile_frames sees valid cfp->pc and the + // ccall doesn't clobber caller-saved registers + // holding stack/local operands. + compile_exit_save_state(asm, exit); + if trace_reason.is_some() || exit.recompile.is_some() { + // Clear cfp->jit_return to prepare for a C call. Normally, cfp->jit_return + // is cleared by the materialize_exit trampoline, but if we're about to + // make a C call, we need to clear any stale JITFrame. + asm_comment!(asm, "clear cfp->jit_return"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + } + if let Some(reason) = trace_reason { + // Leak a CString with the reason so it's available at runtime + let reason_cstr = std::ffi::CString::new(reason.to_string()) + .unwrap_or_else(|_| std::ffi::CString::new("unknown").unwrap()); + let reason_ptr = reason_cstr.into_raw() as *const u8; + asm_ccall!(asm, rb_zjit_record_exit_stack, Opnd::const_ptr(reason_ptr)); + } + compile_exit_recompile(asm, exit); + compile_exit_return(asm); + } + + fn join_opnds(opnds: &Vec<Opnd>, delimiter: &str) -> String { + opnds.iter().map(|opnd| format!("{opnd}")).collect::<Vec<_>>().join(delimiter) + } + + // Extract targets first so that we can update instructions while referencing part of them. let mut targets = HashMap::new(); - for (idx, insn) in self.insns.iter().enumerate() { - if let Some(target @ Target::SideExit { .. }) = insn.target() { - targets.insert(idx, target.clone()); + + for block_id in self.block_order() { + let block = &self.basic_blocks[block_id.0]; + for (idx, insn) in block.insns.iter().enumerate() { + if let Some(target @ Target::SideExit { .. }) = insn.target() { + targets.insert((block_id.0, idx), target.clone()); + } } } - for (idx, target) in targets { - // Compile a side exit. Note that this is past the split pass and alloc_regs(), - // so you can't use a VReg or an instruction that needs to be split. - if let Target::SideExit { pc, stack, locals, reason, label } = target { - asm_comment!(self, "Exit: {reason}"); - let side_exit_label = if let Some(label) = label { - Target::Label(label) + // Create a dedicated block for exit code. This block is not part of the + // CFG (DUMMY_RPO_INDEX), so it won't be included in block_order() or + // linearize_instructions(). Its instructions are returned to the caller + // for appending after scratch_split. + let saved_block = self.current_block_id; + let exit_block = self.new_block_without_id("side_exits"); + + // Map from SideExit to compiled Label. This table is used to deduplicate side exit code. + let mut compiled_exits: HashMap<SideExit, Label> = HashMap::new(); + + // Start a new perf range for side exits + let perf_symbol = if get_option!(perf) == Some(PerfMap::HIR) { + Some(perf_symbol_range_start(self, "side exit")) + } else { + None + }; + + // Mark the start of side-exit code so we can measure its size + if !targets.is_empty() { + self.pos_marker(move |start_pos, cb| { + let end_pos = cb.get_write_ptr(); + let size = end_pos.as_offset() - start_pos.as_offset(); + crate::stats::incr_counter_by(crate::stats::Counter::side_exit_size, size as u64); + }); + } + + // Measure time spent compiling side-exit LIR + let side_exit_start = std::time::Instant::now(); + + for ((block_id, idx), target) in targets { + // Compile a side exit. Note that this is past register assignment, + // so you can't use an instruction that returns a VReg. + if let Target::SideExit { exit: exit @ SideExit { pc, .. }, reason } = target { + // Only record the exit if `trace_side_exits` is defined and the counter is either the one specified + let should_record_exit = get_option!(trace_side_exits).map(|trace| match trace { + TraceExits::All => true, + TraceExits::Counter(counter) if counter == side_exit_counter(reason) => true, + _ => false, + }).unwrap_or(false); + + // If enabled, instrument exits first, and then jump to a shared exit. + let counted_exit = if get_option!(stats) || should_record_exit || cfg!(test) { + let counted_exit = self.new_label("counted_exit"); + self.write_label(counted_exit.clone()); + asm_comment!(self, "Counted Exit: {reason}"); + + if get_option!(stats) || cfg!(test) { + asm_comment!(self, "increment a side exit counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr(reason)), 1.into()); + + if let SideExitReason::UnhandledYARVInsn(opcode) = reason { + asm_comment!(self, "increment an unhandled YARV insn counter"); + self.incr_counter(Opnd::const_ptr(exit_counter_ptr_for_opcode(opcode)), 1.into()); + } + } + + if should_record_exit { + compile_exit(self, &exit, Some(reason)); + } else { + // If the side exit has already been compiled, jump to it. + // Otherwise, let it fall through and compile the exit next. + if let Some(&exit_label) = compiled_exits.get(&exit) { + self.jmp(Target::Label(exit_label)); + } + } + Some(counted_exit) } else { - self.new_label("side_exit".into()) + None }; - self.write_label(side_exit_label.clone()); - // Restore the PC and the stack for regular side exits. We don't do this for - // side exits right after JIT-to-JIT calls, which restore them before the call. - asm_comment!(self, "write stack slots: {stack:?}"); - for (idx, &opnd) in stack.iter().enumerate() { - self.store(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), opnd); + // Compile the shared side exit if not compiled yet + let compiled_exit = if let Some(&compiled_exit) = compiled_exits.get(&exit) { + Target::Label(compiled_exit) + } else { + let new_exit = self.new_label("side_exit"); + self.write_label(new_exit.clone()); + asm_comment!(self, "Exit: {pc}"); + compile_exit(self, &exit, None); + compiled_exits.insert(exit, new_exit.unwrap_label()); + new_exit + }; + + *self.basic_blocks[block_id].insns[idx].target_mut().unwrap() = counted_exit.unwrap_or(compiled_exit); + } + } + + // Measure time spent compiling side-exit LIR + if !compiled_exits.is_empty() { + let nanos = side_exit_start.elapsed().as_nanos(); + crate::stats::incr_counter_by(crate::stats::Counter::compile_side_exit_time_ns, nanos as u64); + crate::stats::incr_counter_by(crate::stats::Counter::compiled_side_exit_count, compiled_exits.len() as u64); + } + + // Close the current perf range for side exits + if let Some(perf_symbol) = &perf_symbol { + perf_symbol_range_end(self, perf_symbol); + } + + // Extract exit instructions and restore the previous current block + let exit_insns = take(&mut self.basic_blocks[exit_block.0].insns); + self.set_current_block(saved_block); + exit_insns + } + + /// Return a traversal of the block graph in reverse post-order. + pub fn reverse_post_order(&self) -> Vec<BlockId> { + let entry_blocks: Vec<BlockId> = self.basic_blocks.iter() + .filter(|block| block.is_entry) + .map(|block| block.id) + .collect(); + let mut result = self.po_from(entry_blocks); + result.reverse(); + result + } + + /// Compute postorder traversal starting from the given blocks. + /// Outbound edges are extracted from the last 0, 1, or 2 instructions (jumps). + fn po_from(&self, starts: Vec<BlockId>) -> Vec<BlockId> { + #[derive(PartialEq)] + enum Action { + VisitEdges, + VisitSelf, + } + let mut result = vec![]; + let mut seen = HashSet::with_capacity(self.basic_blocks.len()); + let mut stack: Vec<_> = starts.iter().map(|&start| (start, Action::VisitEdges)).collect(); + while let Some((block, action)) = stack.pop() { + if action == Action::VisitSelf { + result.push(block); + continue; + } + if !seen.insert(block) { continue; } + stack.push((block, Action::VisitSelf)); + let EdgePair(edge1, edge2) = self.basic_blocks[block.0].edges(); + // Push edge2 before edge1 so that edge1 is popped first from the + // LIFO stack, matching the visit order of a recursive DFS. + if let Some(edge) = edge2 { + stack.push((edge.target, Action::VisitEdges)); + } + if let Some(edge) = edge1 { + stack.push((edge.target, Action::VisitEdges)); + } + } + result + } + + /// Number all instructions in the LIR in reverse postorder. + /// This assigns a unique InsnId to each instruction across all blocks, skipping labels. + /// Also sets the from/to range on each block. + /// Returns the next available instruction ID after numbering. + pub fn number_instructions(&mut self, start: usize) -> usize { + let block_ids = self.block_order(); + let mut insn_id = start; + for block_id in block_ids { + let block = &mut self.basic_blocks[block_id.0]; + let block_start = insn_id; + insn_id += 2; + for (insn, id_slot) in block.insns.iter().zip(block.insn_ids.iter_mut()) { + if matches!(insn, Insn::Label(_)) { + *id_slot = Some(InsnId(block_start)); + } else { + *id_slot = Some(InsnId(insn_id)); + insn_id += 2; + } + } + block.from = InsnId(block_start); + block.to = InsnId(insn_id); + } + insn_id + } + + /// Iterate over all instructions mutably with their block ID, instruction ID, and instruction index within the block. + /// Returns an iterator of (BlockId, `Option<InsnId>`, usize, &mut Insn). + pub fn iter_insns_mut(&mut self) -> impl Iterator<Item = (BlockId, Option<InsnId>, usize, &mut Insn)> { + self.basic_blocks.iter_mut().flat_map(|block| { + let block_id = block.id; + block.insns.iter_mut() + .zip(block.insn_ids.iter().copied()) + .enumerate() + .map(move |(idx, (insn, insn_id))| (block_id, insn_id, idx, insn)) + }) + } + + /// Compute initial liveness sets (kill and gen) for the given blocks. + /// Returns (kill_sets, gen_sets) where each is indexed by block ID. + /// - kill: VRegs defined (written) in the block + /// - gen: VRegs used (read) in the block before being defined + pub fn compute_initial_liveness_sets(&self, block_ids: &[BlockId]) -> (Vec<BitSet<usize>>, Vec<BitSet<usize>>) { + let num_blocks = self.basic_blocks.len(); + let num_vregs = self.num_vregs; + + let mut kill_sets: Vec<BitSet<usize>> = vec![BitSet::with_capacity(num_vregs); num_blocks]; + let mut gen_sets: Vec<BitSet<usize>> = vec![BitSet::with_capacity(num_vregs); num_blocks]; + + for &block_id in block_ids { + let block = &self.basic_blocks[block_id.0]; + let kill_set = &mut kill_sets[block_id.0]; + let gen_set = &mut gen_sets[block_id.0]; + + // Iterate over instructions in reverse + for insn in block.insns.iter().rev() { + // If the instruction has an output that is a VReg, add to kill set + if let Some(out) = insn.out_opnd() { + if let Opnd::VReg { idx, .. } = out { + kill_set.insert(idx.0); + } } - asm_comment!(self, "write locals: {locals:?}"); - for (idx, &opnd) in locals.iter().enumerate() { - self.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); + // For all input operands that are VRegs (including memory base VRegs), add to gen set + insn.for_each_operand(|opnd| { + for idx in opnd.vreg_ids() { + assert!(!kill_set.get(idx.0)); + gen_set.insert(idx.0); + } + }); + } + + // Add block parameters to kill set + for param in &block.parameters { + if let Opnd::VReg { idx, .. } = param { + kill_set.insert(idx.0); + } + } + + } + + (kill_sets, gen_sets) + } + + pub fn block_order(&self) -> Vec<BlockId> { + self.reverse_post_order() + } + + /// Calculate live intervals for each VReg. + pub fn build_intervals(&self, live_in: Vec<BitSet<usize>>) -> Vec<Interval> { + let num_vregs = self.num_vregs; + let mut intervals: Vec<Interval> = (0..num_vregs) + .map(|i| Interval::new(i)) + .collect(); + + let blocks = self.block_order(); + + for block_id in blocks { + let block = &self.basic_blocks[block_id.0]; + + // live = union of successor.liveIn for each successor + let mut live = BitSet::with_capacity(num_vregs); + for succ_id in block.successors() { + live.union_with(&live_in[succ_id.0]); + } + + // Add out_vregs to live set + for idx in block.out_vregs() { + live.insert(idx.0); + } + + // For each live vreg, add entire block range + // block.to is the first instruction of the next block + for idx in live.iter_set_bits() { + intervals[idx].add_range(block.from.0, block.to.0); + } + + // Iterate instructions in reverse + for (insn_id, insn) in block.insn_ids.iter().zip(&block.insns).rev() { + // TODO(max): Remove labels, which are not numbered, in favor of blocks + let Some(insn_id) = insn_id else { continue; }; + // If instruction has VReg output, set_from + if let Some(out) = insn.out_opnd() { + if let Opnd::VReg { idx, .. } = out { + intervals[idx.0].set_from(insn_id.0); + } + } + + // For each VReg input (including memory base VRegs), add_range from block start to insn + insn.for_each_operand(|opnd| { + for idx in opnd.vreg_ids() { + intervals[idx.0].add_range(block.from.0, insn_id.0); + } + }); + } + } + + intervals + } + + /// Analyze liveness for all blocks using a fixed-point algorithm. + /// Returns live_in sets for each block, indexed by block ID. + /// A VReg is live-in to a block if it may be used before being defined. + pub fn analyze_liveness(&self) -> Vec<BitSet<usize>> { + // Get blocks in postorder + let po_blocks = { + let entry_blocks: Vec<BlockId> = self.basic_blocks.iter() + .filter(|block| block.is_entry) + .map(|block| block.id) + .collect(); + self.po_from(entry_blocks) + }; + + // Compute initial gen/kill sets + let (kill_sets, gen_sets) = self.compute_initial_liveness_sets(&po_blocks); + + let num_blocks = self.basic_blocks.len(); + let num_vregs = self.num_vregs; + + // Initialize live_in sets + let mut live_in: Vec<BitSet<usize>> = vec![BitSet::with_capacity(num_vregs); num_blocks]; + + // Fixed-point iteration + let mut changed = true; + while changed { + changed = false; + + // Iterate over blocks in postorder + for &block_id in &po_blocks { + let block = &self.basic_blocks[block_id.0]; + + // block_live = union of live_in[succ] for all successors + let mut block_live = BitSet::with_capacity(num_vregs); + for succ_id in block.successors() { + block_live.union_with(&live_in[succ_id.0]); + } + + // block_live |= gen[block] + block_live.union_with(&gen_sets[block_id.0]); + + // block_live &= ~kill[block] + block_live.difference_with(&kill_sets[block_id.0]); + + // Update live_in if changed + if !live_in[block_id.0].equals(&block_live) { + live_in[block_id.0] = block_live; + changed = true; + } + } + } + + live_in + } +} + +/// Return a result of fmt::Display for Assembler without escape sequence +pub fn lir_string(asm: &Assembler) -> String { + use crate::ttycolors::TTY_TERMINAL_COLOR; + format!("{asm}").replace(TTY_TERMINAL_COLOR.bold_begin, "").replace(TTY_TERMINAL_COLOR.bold_end, "") +} + +/// Format live intervals as a grid showing which VRegs are alive at each instruction +pub fn lir_intervals_string(asm: &Assembler, intervals: &[Interval]) -> String { + let mut output = String::new(); + let num_vregs = intervals.len(); + + let vreg_header = |output: &mut String| { + output.push_str(" "); + for i in 0..num_vregs { + output.push_str(&format!(" v{:<2}", i)); + } + output.push('\n'); + + output.push_str(" "); + for _ in 0..num_vregs { + output.push_str(" ---"); + } + output.push('\n'); + }; + + // Collect all numbered instruction positions in RPO order + let mut first = true; + for block_id in asm.block_order() { + let block = &asm.basic_blocks[block_id.0]; + + // Print VReg header before each block + if !first { output.push('\n'); } + first = false; + vreg_header(&mut output); + + // Print basic block label header with parameters + let label = asm.block_label(block_id); + if block.parameters.is_empty() { + output.push_str(&format!("{}():\n", asm.label_names[label.0])); + } else { + output.push_str(&format!("{}(", asm.label_names[label.0])); + for (idx, param) in block.parameters.iter().enumerate() { + if idx > 0 { + output.push_str(", "); + } + output.push_str(&format!("{param}")); + } + output.push_str("):\n"); + } + + for (insn, insn_id) in block.insns.iter().zip(&block.insn_ids) { + // Skip labels (they're not numbered) + let Some(insn_id) = insn_id else { panic!("{insn:?}"); }; + + // Print instruction ID + output.push_str(&format!("i{:<6}: ", insn_id.0)); + + // For each VReg, check if it's alive at this position + for vreg_idx in 0..num_vregs { + let is_alive = intervals[vreg_idx].range.start.is_some() && + intervals[vreg_idx].range.end.is_some() && + intervals[vreg_idx].survives(insn_id.0); + + let has_range = intervals[vreg_idx].range.start.is_some(); + if has_range && intervals[vreg_idx].born_at(insn_id.0) { + output.push_str(" v "); + } else if has_range && intervals[vreg_idx].dies_at(insn_id.0) { + output.push_str(" ^ "); + } else if is_alive { + output.push_str(" █ "); + } else { + output.push_str(" . "); } + } - asm_comment!(self, "save cfp->pc"); - self.load_into(Opnd::Reg(Assembler::SCRATCH_REG), Opnd::const_ptr(pc)); - self.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::Reg(Assembler::SCRATCH_REG)); + if let Insn::Label(_) = insn { + output.push('\n'); + continue; + } - asm_comment!(self, "save cfp->sp"); - self.lea_into(Opnd::Reg(Assembler::SCRATCH_REG), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); - let cfp_sp = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); - self.store(cfp_sp, Opnd::Reg(Assembler::SCRATCH_REG)); + // Show the instruction text using compact formatting + output.push_str(" "); - asm_comment!(self, "exit to the interpreter"); - self.frame_teardown(&[]); // matching the setup in :bb0-prologue: - self.mov(C_RET_OPND, Opnd::UImm(Qundef.as_u64())); - self.cret(C_RET_OPND); + if let Insn::Comment(comment) = insn { + output.push_str(&format!("# {}", comment)); + } else { + // Print output operand if any + if let Some(out) = insn.out_opnd() { + output.push_str(&format!("{out} = ")); + } - *self.insns[idx].target_mut().unwrap() = side_exit_label; + // Use the helper function to format instruction (reuses Display logic) + output.push_str(&format_insn_compact(asm, insn)); } + + output.push('\n'); } } + + output +} + +/// Format live intervals as a grid showing which VRegs are alive at each instruction +pub fn debug_intervals(asm: &Assembler, intervals: &[Interval]) -> String { + lir_intervals_string(asm, intervals) +} + +/// Helper function to format a single instruction (without the output part, which is already printed) +/// Returns a string formatted like: "OpName target operand1, operand2, ..." +fn format_insn_compact(asm: &Assembler, insn: &Insn) -> String { + let mut output = String::new(); + + // Print the instruction name + output.push_str(insn.op()); + + // Print target (before operands, to match --zjit-dump-lir format) + if let Some(target) = insn.target() { + match target { + Target::CodePtr(code_ptr) => output.push_str(&format!(" {code_ptr:?}")), + Target::Label(Label(label_idx)) => output.push_str(&format!(" {}", asm.label_names[*label_idx])), + Target::SideExit { reason, .. } => output.push_str(&format!(" Exit({reason})")), + Target::Block(edge) => { + let label = asm.block_label(edge.target); + let name = &asm.label_names[label.0]; + if edge.args.is_empty() { + output.push_str(&format!(" {name}")); + } else { + output.push_str(&format!(" {name}(")); + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + output.push_str(", "); + } + output.push_str(&format!("{}", arg)); + } + output.push_str(")"); + } + } + } + } + + // Print operands (but skip branch args since they're already printed with target) + if let Some(Target::SideExit { .. }) = insn.target() { + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + output.push_str(&format!(", {opnd}")); + } + _ => {} + } + } else if let Some(Target::Block(_)) = insn.target() { + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + output.push_str(&format!(", {opnd}")); + } + _ => {} + } + } else if insn.opnd_count() > 0 { + let mut sep = ""; + insn.for_each_operand(|opnd| { + output.push_str(&format!("{sep}{opnd}")); + sep = ", "; + }); + } + + output +} + +impl fmt::Display for Assembler { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Count the number of duplicated label names to disambiguate them if needed + let mut label_counts: HashMap<&String, usize> = HashMap::new(); + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + for label_name in self.label_names.iter() { + let counter = label_counts.entry(label_name).or_insert(0); + *counter += 1; + } + + /// Return a label name String. Suffix "_{label_idx}" if the label name is used multiple times. + fn label_name(asm: &Assembler, label_idx: usize, label_counts: &HashMap<&String, usize>) -> String { + let label_name = &asm.label_names[label_idx]; + let label_count = label_counts.get(&label_name).unwrap_or(&0); + if *label_count > 1 { + format!("{label_name}_{label_idx}") + } else { + label_name.to_string() + } + } + + // Use sorted_blocks() instead of block_order() because block_order() + // calls rpo() -> edges() which requires all blocks end with terminators. + // After arm64_scratch_split, blocks may not have terminators. + for bb in self.sorted_blocks() { + let params = &bb.parameters; + for (insn_id, insn) in bb.insn_ids.iter().zip(&bb.insns) { + if let Some(id) = insn_id { + write!(f, "{id}: ")?; + } else { + write!(f, " ")?; + } + match insn { + Insn::Comment(comment) => { + writeln!(f, " {bold_begin}# {comment}{bold_end}")?; + } + Insn::Label(target) => { + let Target::Label(Label(label_idx)) = target else { + panic!("unexpected target for Insn::Label: {target:?}"); + }; + write!(f, " {}(", label_name(self, *label_idx, &label_counts))?; + for (idx, param) in params.iter().enumerate() { + if idx > 0 { + write!(f, ", ")?; + } + write!(f, "{param}")?; + } + writeln!(f, "):")?; + } + _ => { + write!(f, " ")?; + + // Print output operand if any + if let Some(out) = insn.out_opnd() { + write!(f, "{out} = ")?; + } + + // Print the instruction name + write!(f, "{}", insn.op())?; + + // Show slot_count for FrameSetup + if let Insn::FrameSetup { slot_count, preserved } = insn { + write!(f, " {slot_count}")?; + if !preserved.is_empty() { + write!(f, ",")?; + } + } + + // Print target + if let Some(target) = insn.target() { + match target { + Target::CodePtr(code_ptr) => write!(f, " {code_ptr:?}")?, + Target::Label(Label(label_idx)) => write!(f, " {}", label_name(self, *label_idx, &label_counts))?, + Target::SideExit { reason, .. } => write!(f, " Exit({reason})")?, + Target::Block(edge) => { + let label = self.block_label(edge.target); + let name = label_name(self, label.0, &label_counts); + if edge.args.is_empty() { + write!(f, " {name}")?; + } else { + write!(f, " {name}(")?; + for (i, arg) in edge.args.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", arg)?; + } + write!(f, ")")?; + } + } + } + } + + // Print list of operands + if let Some(Target::SideExit { .. }) = insn.target() { + // If the instruction has a SideExit, avoid using opnd_iter(), which has stack/locals. + // Here, only handle instructions that have both Opnd and Target. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } + } else if let Some(Target::Block(_)) = insn.target() { + // If the instruction has a Block target, avoid using opnd_iter() for branch args + // since they're already printed inline with the target. Only print non-target operands. + match insn { + Insn::Joz(opnd, _) | + Insn::Jonz(opnd, _) | + Insn::LeaJumpTarget { out: opnd, target: _ } => { + write!(f, ", {opnd}")?; + } + _ => {} + } + } else if insn.opnd_count() > 0 { + let mut sep = " "; + insn.try_for_each_operand(|opnd| { + let result = write!(f, "{sep}{opnd}"); + sep = ", "; + result + })?; + } + + write!(f, "\n")?; + } + } + } + } + Ok(()) + } } impl fmt::Debug for Assembler { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { writeln!(fmt, "Assembler")?; - for (idx, insn) in self.insns.iter().enumerate() { + for (idx, insn) in self.linearize_instructions().iter().enumerate() { writeln!(fmt, " {idx:03} {insn:?}")?; } @@ -1615,6 +3121,64 @@ impl fmt::Debug for Assembler { } } +pub struct InsnIter { + blocks: Vec<BasicBlock>, + current_block_idx: usize, + current_insn_iter: std::vec::IntoIter<Insn>, + peeked: Option<(usize, Insn)>, + index: usize, +} + +impl InsnIter { + // We're implementing our own peek() because we don't want peek to + // cross basic blocks as we're iterating. + pub fn peek(&mut self) -> Option<&(usize, Insn)> { + // If we don't have a peeked value, get one + if self.peeked.is_none() { + let insn = self.current_insn_iter.next()?; + let idx = self.index; + self.index += 1; + self.peeked = Some((idx, insn)); + } + // Return a reference to the peeked value + self.peeked.as_ref() + } + + // Get the next instruction, advancing to the next block when current block is exhausted. + // Sets the current block on new_asm when moving to a new block. + pub fn next(&mut self, new_asm: &mut Assembler) -> Option<(usize, Insn)> { + // If we have a peeked value, return it + if let Some(item) = self.peeked.take() { + return Some(item); + } + + // Try to get the next instruction from current block + if let Some(insn) = self.current_insn_iter.next() { + let idx = self.index; + self.index += 1; + return Some((idx, insn)); + } + + // Current block is exhausted, move to next block + self.current_block_idx += 1; + if self.current_block_idx >= self.blocks.len() { + return None; + } + + // Set up the next block + let next_block = &mut self.blocks[self.current_block_idx]; + new_asm.set_current_block(next_block.id); + + self.current_insn_iter = take(&mut next_block.insns).into_iter(); + + // Get first instruction from the new block + let insn = self.current_insn_iter.next()?; + let idx = self.index; + self.index += 1; + Some((idx, insn)) + } +} + impl Assembler { #[must_use] pub fn add(&mut self, left: Opnd, right: Opnd) -> Opnd { @@ -1639,15 +3203,42 @@ impl Assembler { self.push_insn(Insn::BakeString(text.to_string())); } + pub fn is_ruby_code(&self) -> bool { + self.basic_blocks.len() > 1 || !self.basic_blocks[0].is_dummy() + } + #[allow(dead_code)] pub fn breakpoint(&mut self) { self.push_insn(Insn::Breakpoint); } + #[allow(dead_code)] + pub fn abort(&mut self) { + self.push_insn(Insn::Abort); + } + /// Call a C function without PosMarkers pub fn ccall(&mut self, fptr: *const u8, opnds: Vec<Opnd>) -> Opnd { + let canary_opnd = self.set_stack_canary(); let out = self.new_vreg(Opnd::match_num_bits(&opnds)); + let fptr = Opnd::const_ptr(fptr); self.push_insn(Insn::CCall { fptr, opnds, start_marker: None, end_marker: None, out }); + self.clear_stack_canary(canary_opnd); + out + } + + /// Call a C function into an explicit output operand without allocating a + /// new vreg for the result. + pub fn ccall_into(&mut self, out: Opnd, fptr: *const u8, opnds: Vec<Opnd>) { + let fptr = Opnd::const_ptr(fptr); + self.push_insn(Insn::CCall { fptr, opnds, start_marker: None, end_marker: None, out }); + } + + /// Call a C function stored in a register + pub fn ccall_reg(&mut self, fptr: Opnd, num_bits: u8) -> Opnd { + assert!(matches!(fptr, Opnd::Reg(_)), "ccall_reg must be called with Opnd::Reg: {fptr:?}"); + let out = self.new_vreg(num_bits); + self.push_insn(Insn::CCall { fptr, opnds: vec![], start_marker: None, end_marker: None, out }); out } @@ -1662,15 +3253,26 @@ impl Assembler { ) -> Opnd { let out = self.new_vreg(Opnd::match_num_bits(&opnds)); self.push_insn(Insn::CCall { - fptr, + fptr: Opnd::const_ptr(fptr), opnds, - start_marker: Some(Box::new(start_marker)), - end_marker: Some(Box::new(end_marker)), + start_marker: Some(Rc::new(start_marker)), + end_marker: Some(Rc::new(end_marker)), out, }); out } + pub fn count_call_to(&mut self, fn_name: &str) { + // We emit ccalls while initializing the JIT. Unfortunately, we skip those because + // otherwise we have no counter pointers to read. + if crate::state::ZJITState::has_instance() && get_option!(stats) { + let ccall_counter_pointers = crate::state::ZJITState::get_ccall_counter_pointers(); + let counter_ptr = ccall_counter_pointers.entry(fn_name.to_string()).or_insert_with(|| Box::new(0)); + let counter_ptr: &mut u64 = counter_ptr.as_mut(); + self.incr_counter(Opnd::const_ptr(counter_ptr), 1.into()); + } + } + pub fn cmp(&mut self, left: Opnd, right: Opnd) { self.push_insn(Insn::Cmp { left, right }); } @@ -1682,21 +3284,27 @@ impl Assembler { out } - pub fn cpop_all(&mut self) { - self.push_insn(Insn::CPopAll); - } - pub fn cpop_into(&mut self, opnd: Opnd) { assert!(matches!(opnd, Opnd::Reg(_)), "Destination of cpop_into must be a register, got: {opnd:?}"); self.push_insn(Insn::CPopInto(opnd)); } + #[track_caller] + pub fn cpop_pair_into(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpop_pair_into must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPopPairInto(opnd0, opnd1)); + } + pub fn cpush(&mut self, opnd: Opnd) { self.push_insn(Insn::CPush(opnd)); } - pub fn cpush_all(&mut self) { - self.push_insn(Insn::CPushAll); + #[track_caller] + pub fn cpush_pair(&mut self, opnd0: Opnd, opnd1: Opnd) { + assert!(matches!(opnd0, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd0:?}"); + assert!(matches!(opnd1, Opnd::Reg(_) | Opnd::VReg{ .. }), "Destination of cpush_pair must be a register, got: {opnd1:?}"); + self.push_insn(Insn::CPushPair(opnd0, opnd1)); } pub fn cret(&mut self, opnd: Opnd) { @@ -1759,7 +3367,8 @@ impl Assembler { out } - pub fn frame_setup(&mut self, preserved_regs: &'static [Opnd], slot_count: usize) { + pub fn frame_setup(&mut self, preserved_regs: &'static [Opnd]) { + let slot_count = self.stack_base_idx; self.push_insn(Insn::FrameSetup { preserved: preserved_regs, slot_count }); } @@ -1773,32 +3382,15 @@ impl Assembler { self.push_insn(Insn::IncrCounter { mem, value }); } - pub fn jbe(&mut self, target: Target) { - self.push_insn(Insn::Jbe(target)); - } - pub fn jb(&mut self, target: Target) { self.push_insn(Insn::Jb(target)); } - pub fn je(&mut self, target: Target) { - self.push_insn(Insn::Je(target)); - } - - pub fn jl(&mut self, target: Target) { - self.push_insn(Insn::Jl(target)); - } - #[allow(dead_code)] pub fn jg(&mut self, target: Target) { self.push_insn(Insn::Jg(target)); } - #[allow(dead_code)] - pub fn jge(&mut self, target: Target) { - self.push_insn(Insn::Jge(target)); - } - pub fn jmp(&mut self, target: Target) { self.push_insn(Insn::Jmp(target)); } @@ -1807,25 +3399,6 @@ impl Assembler { self.push_insn(Insn::JmpOpnd(opnd)); } - pub fn jne(&mut self, target: Target) { - self.push_insn(Insn::Jne(target)); - } - - pub fn jnz(&mut self, target: Target) { - self.push_insn(Insn::Jnz(target)); - } - - pub fn jo(&mut self, target: Target) { - self.push_insn(Insn::Jo(target)); - } - - pub fn jo_mul(&mut self, target: Target) { - self.push_insn(Insn::JoMul(target)); - } - - pub fn jz(&mut self, target: Target) { - self.push_insn(Insn::Jz(target)); - } #[must_use] pub fn lea(&mut self, opnd: Opnd) -> Opnd { @@ -1835,7 +3408,7 @@ impl Assembler { } pub fn lea_into(&mut self, out: Opnd, opnd: Opnd) { - assert!(matches!(out, Opnd::Reg(_)), "Destination of lea_into must be a register, got: {out:?}"); + assert!(matches!(out, Opnd::Reg(_) | Opnd::Mem(_)), "Destination of lea_into must be a register or memory, got: {out:?}"); self.push_insn(Insn::Lea { opnd, out }); } @@ -1847,13 +3420,6 @@ impl Assembler { } #[must_use] - pub fn live_reg_opnd(&mut self, opnd: Opnd) -> Opnd { - let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); - self.push_insn(Insn::LiveReg { opnd, out }); - out - } - - #[must_use] pub fn load(&mut self, opnd: Opnd) -> Opnd { let out = self.new_vreg(Opnd::match_num_bits(&[opnd])); self.push_insn(Insn::Load { opnd, out }); @@ -1882,10 +3448,6 @@ impl Assembler { out } - pub fn parallel_mov(&mut self, moves: Vec<(Reg, Opnd)>) { - self.push_insn(Insn::ParallelMov { moves }); - } - pub fn mov(&mut self, dest: Opnd, src: Opnd) { assert!(!matches!(dest, Opnd::VReg { .. }), "Destination of mov must not be Opnd::VReg, got: {dest:?}"); self.push_insn(Insn::Mov { dest, src }); @@ -1905,8 +3467,8 @@ impl Assembler { out } - pub fn patch_point(&mut self, target: Target) { - self.push_insn(Insn::PatchPoint(target)); + pub fn patch_point(&mut self, target: Target, invariant: Invariant, version: IseqVersionRef) { + self.push_insn(Insn::PatchPoint { target, invariant, version }); } pub fn pad_patch_point(&mut self) { @@ -1914,7 +3476,7 @@ impl Assembler { } pub fn pos_marker(&mut self, marker_fn: impl Fn(CodePtr, &CodeBlock) + 'static) { - self.push_insn(Insn::PosMarker(Box::new(marker_fn))); + self.push_insn(Insn::PosMarker(Rc::new(marker_fn))); } #[must_use] @@ -1972,13 +3534,51 @@ impl Assembler { self.push_insn(Insn::Xor { left, right, out }); out } + + /// This is used for trampolines that don't allow scratch registers. + /// Linearizes all blocks into a single giant block. + pub fn resolve_parallel_mov_pass(self) -> Assembler { + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = self.accept_scratch_reg; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.num_vregs = self.num_vregs; + + // Create one giant block to linearize everything into + asm_local.new_block_without_id("linearized"); + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + // TODO: Aaron, this could be better. We don't need to do this, FIXME + // Process each linearized instruction + for insn in linearized_insns { + match insn { + Insn::Mov { dest, src } => { + if src != dest { + asm_local.push_insn(insn); + } + }, + _ => { + asm_local.push_insn(insn); + } + } + } + + asm_local + } } /// Macro to use format! for Insn::Comment, which skips a format! call /// when not dumping disassembly. macro_rules! asm_comment { ($asm:expr, $($fmt:tt)*) => { - if $crate::options::get_option!(dump_disasm) { + // If --zjit-dump-disasm or --zjit-dump-lir is given, enrich them with comments. + // Also allow --zjit-debug on dev builds to enable comments since dev builds dump LIR on panic. + let enable_comment = $crate::options::get_option_ref!(dump_disasm).is_some() || + $crate::options::get_option!(dump_lir).is_some() || + (cfg!(debug_assertions) && $crate::options::get_option!(debug)); + if enable_comment { $asm.push_insn(crate::backend::lir::Insn::Comment(format!($($fmt)*))); } }; @@ -1989,6 +3589,7 @@ pub(crate) use asm_comment; macro_rules! asm_ccall { [$asm: ident, $fn_name:ident, $($args:expr),* ] => {{ $crate::backend::lir::asm_comment!($asm, concat!("call ", stringify!($fn_name))); + $asm.count_call_to(stringify!($fn_name)); $asm.ccall($fn_name as *const u8, vec![$($args),*]) }}; } @@ -1997,27 +3598,35 @@ pub(crate) use asm_ccall; #[cfg(test)] mod tests { use super::*; + use insta::assert_snapshot; + use crate::backend::current::NATIVE_STACK_PTR; + + fn scratch_reg() -> Opnd { + Assembler::new_with_scratch_reg().1 + } #[test] - fn test_opnd_iter() { + fn test_for_each_operand() { let insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; - let mut opnd_iter = insn.opnd_iter(); - assert!(matches!(opnd_iter.next(), Some(Opnd::None))); - assert!(matches!(opnd_iter.next(), Some(Opnd::None))); - - assert!(matches!(opnd_iter.next(), None)); + let mut result = vec![]; + insn.for_each_operand(|opnd| result.push(opnd)); + assert_eq!(result, vec![Opnd::None, Opnd::None]); } #[test] - fn test_opnd_iter_mut() { + fn test_for_each_operand_mut() { let mut insn = Insn::Add { left: Opnd::None, right: Opnd::None, out: Opnd::None }; - let mut opnd_iter = insn.opnd_iter_mut(); - assert!(matches!(opnd_iter.next(), Some(Opnd::None))); - assert!(matches!(opnd_iter.next(), Some(Opnd::None))); - - assert!(matches!(opnd_iter.next(), None)); + let mut counter = 0; + insn.for_each_operand_mut(|opnd| { + *opnd = Opnd::Imm(counter); + counter += 1; + }); + assert!(matches!(insn, Insn::Add { left: Opnd::Imm(0), right: Opnd::Imm(1), out: Opnd::None })); + let mut result = vec![]; + insn.for_each_operand(|opnd| result.push(opnd)); + assert_eq!(result, vec![Opnd::Imm(0), Opnd::Imm(1)]); } #[test] @@ -2027,5 +3636,837 @@ mod tests { let mem = Opnd::mem(64, SP, 0); asm.load_into(mem, mem); } -} + #[test] + fn test_resolve_parallel_moves_reorder_registers() { + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, Some(vec![ + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + fn test_resolve_parallel_moves_give_up_register_cycle() { + // If scratch_opnd is not given, it cannot break cycles. + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], None); + assert_eq!(result, None); + } + + #[test] + fn test_resolve_parallel_moves_break_register_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], C_ARG_OPNDS[0]), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_memory_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_break_register_memory_cycle() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (scratch_reg, C_ARG_OPNDS[1]), + (C_ARG_OPNDS[1], Opnd::mem(64, C_ARG_OPNDS[0], 0)), + (C_ARG_OPNDS[0], scratch_reg), + ])); + } + + #[test] + fn test_resolve_parallel_moves_reorder_memory_destination() { + let scratch_reg = scratch_reg(); + let result = Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg)); + assert_eq!(result, Some(vec![ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + (C_ARG_OPNDS[0], SP), + ])); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_register() { + Assembler::resolve_parallel_moves(&[ + (C_ARG_OPNDS[0], SP), + (C_ARG_OPNDS[0], CFP), + ], Some(scratch_reg())); + } + + #[test] + #[should_panic] + fn test_resolve_parallel_moves_into_same_memory() { + Assembler::resolve_parallel_moves(&[ + (Opnd::mem(64, C_ARG_OPNDS[0], 0), SP), + (Opnd::mem(64, C_ARG_OPNDS[0], 0), CFP), + ], Some(scratch_reg())); + } + + // Helper function to convert a BitSet to a list of vreg indices + fn bitset_to_vreg_indices(bitset: &BitSet<usize>, num_vregs: usize) -> Vec<usize> { + (0..num_vregs) + .filter(|&idx| bitset.get(idx)) + .collect() + } + + struct TestFunc { + asm: Assembler, + r10: Opnd, + r11: Opnd, + r12: Opnd, + r13: Opnd, + r14: Opnd, + r15: Opnd, + b1: BlockId, + b2: BlockId, + b3: BlockId, + b4: BlockId, + } + + fn build_func() -> TestFunc { + let mut asm = Assembler::new(); + + // Create virtual registers - these will be parameters + let r10 = asm.new_vreg(64); + let r11 = asm.new_vreg(64); + let r12 = asm.new_vreg(64); + let r13 = asm.new_vreg(64); + + // Create blocks + let b1 = asm.new_block(hir::BlockId(0), true, 0); + let b2 = asm.new_block(hir::BlockId(1), false, 1); + let b3 = asm.new_block(hir::BlockId(2), false, 2); + let b4 = asm.new_block(hir::BlockId(3), false, 3); + + // Build b1: define(r10, r11) { jump(edge(b2, [imm(1), r11])) } + asm.set_current_block(b1); + let label_b1 = asm.new_label("bb0"); + asm.write_label(label_b1); + asm.basic_blocks[b1.0].add_parameter(r10); + asm.basic_blocks[b1.0].add_parameter(r11); + asm.basic_blocks[b1.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: b2, + args: vec![Opnd::UImm(1), r11], + }))); + + // Build b2: define(r12, r13) { cmp(r13, imm(1)); blt(...) } + asm.set_current_block(b2); + let label_b2 = asm.new_label("bb1"); + asm.write_label(label_b2); + asm.basic_blocks[b2.0].add_parameter(r12); + asm.basic_blocks[b2.0].add_parameter(r13); + asm.basic_blocks[b2.0].push_insn(Insn::Cmp { left: r13, right: Opnd::UImm(1) }); + asm.basic_blocks[b2.0].push_insn(Insn::Jl(Target::Block(BranchEdge { target: b4, args: vec![] }))); + asm.basic_blocks[b2.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { target: b3, args: vec![] }))); + + // Build b3: r14 = mul(r12, r13); r15 = sub(r13, imm(1)); jump(edge(b2, [r14, r15])) + asm.set_current_block(b3); + let label_b3 = asm.new_label("bb2"); + asm.write_label(label_b3); + let r14 = asm.new_vreg(64); + let r15 = asm.new_vreg(64); + asm.basic_blocks[b3.0].push_insn(Insn::Mul { left: r12, right: r13, out: r14 }); + asm.basic_blocks[b3.0].push_insn(Insn::Sub { left: r13, right: Opnd::UImm(1), out: r15 }); + asm.basic_blocks[b3.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: b2, + args: vec![r14, r15], + }))); + + // Build b4: out = add(r10, r12); ret out + asm.set_current_block(b4); + let label_b4 = asm.new_label("bb3"); + asm.write_label(label_b4); + let out = asm.new_vreg(64); + asm.basic_blocks[b4.0].push_insn(Insn::Add { left: r10, right: r12, out }); + asm.basic_blocks[b4.0].push_insn(Insn::CRet(out)); + + TestFunc { asm, r10, r11, r12, r13, r14, r15, b1, b2, b3, b4 } + } + + #[test] + fn test_live_in() { + let TestFunc { asm, r10, r12, r13, b1, b2, b3, b4, .. } = build_func(); + + let num_vregs = asm.num_vregs; + let live_in = asm.analyze_liveness(); + + // b1: [] - entry block, no variables are live-in + assert_eq!(bitset_to_vreg_indices(&live_in[b1.0], num_vregs), vec![]); + + // b2: [r10] - r10 is live-in (used in b4 which is reachable) + assert_eq!(bitset_to_vreg_indices(&live_in[b2.0], num_vregs), vec![r10.vreg_idx().0]); + + // b3: [r10, r12, r13] - all are live-in + assert_eq!( + bitset_to_vreg_indices(&live_in[b3.0], num_vregs), + vec![r10.vreg_idx().0, r12.vreg_idx().0, r13.vreg_idx().0] + ); + + // b4: [r10, r12] - both are live-in + assert_eq!( + bitset_to_vreg_indices(&live_in[b4.0], num_vregs), + vec![r10.vreg_idx().0, r12.vreg_idx().0] + ); + } + + #[test] + fn test_lir_debug_output() { + let TestFunc { asm, .. } = build_func(); + + // Test the LIR string output + let output = lir_string(&asm); + + assert_snapshot!(output, @" + bb0(v0, v1): + Jmp bb1(1, v1) + bb1(v2, v3): + Cmp v3, 1 + Jl bb3 + Jmp bb2 + bb2(): + v4 = Mul v2, v3 + v5 = Sub v3, 1 + Jmp bb1(v4, v5) + bb3(): + v6 = Add v0, v2 + CRet v6 + "); + } + + #[test] + fn test_out_vregs() { + let TestFunc { asm, r11, r14, r15, b1, b2, b3, b4, .. } = build_func(); + + // b1 has one edge to b2 with args [imm(1), r11] + // Only r11 is a VReg, so we should only get that + let out_b1 = asm.basic_blocks[b1.0].out_vregs(); + assert_eq!(out_b1.len(), 1); + assert_eq!(out_b1[0], r11.vreg_idx()); + + // b2 has two edges: one to b4 (no args) and one to b3 (no args) + let out_b2 = asm.basic_blocks[b2.0].out_vregs(); + assert_eq!(out_b2.len(), 0); + + // b3 has one edge to b2 with args [r14, r15] + let out_b3 = asm.basic_blocks[b3.0].out_vregs(); + assert_eq!(out_b3.len(), 2); + assert_eq!(out_b3[0], r14.vreg_idx()); + assert_eq!(out_b3[1], r15.vreg_idx()); + + // b4 has no edges (terminates with CRet) + let out_b4 = asm.basic_blocks[b4.0].out_vregs(); + assert_eq!(out_b4.len(), 0); + } + + #[test] + fn test_out_vregs_includes_memory_base_vregs() { + let mut asm = Assembler::new(); + + let base = asm.new_vreg(64); + let b1 = asm.new_block(hir::BlockId(0), true, 0); + let b2 = asm.new_block(hir::BlockId(1), false, 1); + + asm.set_current_block(b1); + let label_b1 = asm.new_label("bb0"); + asm.write_label(label_b1); + asm.basic_blocks[b1.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { + target: b2, + args: vec![Opnd::mem(64, base, 8)], + }))); + + let out_vregs = asm.basic_blocks[b1.0].out_vregs(); + assert_eq!(out_vregs, vec![base.vreg_idx()]); + } + + #[test] + fn test_interval_add_range() { + let mut interval = Interval::new(1); + + // Add range to empty interval + interval.add_range(5, 10); + assert_eq!(interval.range.start, Some(5)); + assert_eq!(interval.range.end, Some(10)); + + // Extend range backward + interval.add_range(3, 7); + assert_eq!(interval.range.start, Some(3)); + assert_eq!(interval.range.end, Some(10)); + + // Extend range forward + interval.add_range(8, 15); + assert_eq!(interval.range.start, Some(3)); + assert_eq!(interval.range.end, Some(15)); + } + + #[test] + fn test_interval_survives() { + let mut interval = Interval::new(1); + interval.add_range(3, 10); + + assert!(!interval.survives(2)); // Before range + assert!(!interval.survives(3)); // At start (exclusive) + assert!(interval.survives(5)); // Inside range + assert!(!interval.survives(10)); // At end (exclusive) + assert!(!interval.survives(11)); // After range + } + + #[test] + fn test_interval_set_from() { + let mut interval = Interval::new(1); + + // With no range, sets both start and end + interval.set_from(10); + assert_eq!(interval.range.start, Some(10)); + assert_eq!(interval.range.end, Some(10)); + + // With existing range, updates start but keeps end + interval.add_range(5, 20); + interval.set_from(3); + assert_eq!(interval.range.start, Some(3)); + assert_eq!(interval.range.end, Some(20)); + } + + #[test] + #[should_panic(expected = "Invalid range")] + fn test_interval_add_range_invalid() { + let mut interval = Interval::new(1); + interval.add_range(10, 5); + } + + #[test] + #[should_panic(expected = "survives called on interval with no range")] + fn test_interval_survives_panics_without_range() { + let interval = Interval::new(1); + interval.survives(5); + } + + #[test] + fn test_build_intervals() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + // Analyze liveness + let live_in = asm.analyze_liveness(); + + // Number instructions (starting from 16 to match Ruby test) + asm.number_instructions(16); + + // Build intervals + let intervals = asm.build_intervals(live_in); + + // Extract vreg indices + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + // Assert expected ranges + // Note: Rust CFG differs from Ruby due to conditional branches requiring two instructions (Jl + Jmp) + assert_eq!(intervals[r10_idx.0].range.start, Some(16)); + assert_eq!(intervals[r10_idx.0].range.end, Some(38)); + + assert_eq!(intervals[r11_idx.0].range.start, Some(16)); + assert_eq!(intervals[r11_idx.0].range.end, Some(20)); + + assert_eq!(intervals[r12_idx.0].range.start, Some(20)); + assert_eq!(intervals[r12_idx.0].range.end, Some(38)); + + assert_eq!(intervals[r13_idx.0].range.start, Some(20)); + assert_eq!(intervals[r13_idx.0].range.end, Some(32)); + + assert_eq!(intervals[r14_idx.0].range.start, Some(30)); + assert_eq!(intervals[r14_idx.0].range.end, Some(36)); + + assert_eq!(intervals[r15_idx.0].range.start, Some(32)); + assert_eq!(intervals[r15_idx.0].range.end, Some(36)); + } + + #[test] + fn test_linear_scan_no_spill() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + // Analyze liveness + let live_in = asm.analyze_liveness(); + + // Number instructions (starting from 16 to match Ruby test) + asm.number_instructions(16); + + // Build intervals + let intervals = asm.build_intervals(live_in); + + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 5, &preferred_registers); + + // Extract vreg indices + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + // 5 registers is enough for all intervals, no spills needed + assert_eq!(num_stack_slots, 0); + + // Verify register assignments + // r10: [16,42) gets Reg(0) (first allocated) + // r11: [16,20) gets Reg(1) + // r12: [20,36) gets Reg(1) (r11 expired, reuses its register) + // r13: [20,38) gets Reg(2) + // r14: [36,42) gets Reg(1) (r12 expired, reuses its register) + // r15: [38,42) gets Reg(2) (r13 expired, reuses its register) + assert_eq!(assignments[r10_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r11_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r12_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r13_idx.0], Some(Allocation::Reg(2))); + assert_eq!(assignments[r14_idx.0], Some(Allocation::Reg(3))); + assert_eq!(assignments[r15_idx.0], Some(Allocation::Reg(2))); + } + + #[test] + fn test_linear_scan_spill_less() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + + // 3 registers -- only r10 needs to spill + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 3, &preferred_registers); + + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + assert_eq!(num_stack_slots, 1); + assert_eq!(assignments[r10_idx.0], Some(Allocation::Stack(0))); + assert_eq!(assignments[r11_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r12_idx.0], Some(Allocation::Reg(1))); + assert_eq!(assignments[r13_idx.0], Some(Allocation::Reg(2))); + assert_eq!(assignments[r14_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r15_idx.0], Some(Allocation::Reg(2))); + } + + #[test] + fn test_linear_scan_spill() { + let TestFunc { mut asm, r10, r11, r12, r13, r14, r15, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + + // Only 1 register available -- forces spills + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 1, &preferred_registers); + + let r10_idx = if let Opnd::VReg { idx, .. } = r10 { idx } else { panic!() }; + let r11_idx = if let Opnd::VReg { idx, .. } = r11 { idx } else { panic!() }; + let r12_idx = if let Opnd::VReg { idx, .. } = r12 { idx } else { panic!() }; + let r13_idx = if let Opnd::VReg { idx, .. } = r13 { idx } else { panic!() }; + let r14_idx = if let Opnd::VReg { idx, .. } = r14 { idx } else { panic!() }; + let r15_idx = if let Opnd::VReg { idx, .. } = r15 { idx } else { panic!() }; + + assert_eq!(num_stack_slots, 3); + assert_eq!(assignments[r10_idx.0], Some(Allocation::Stack(0))); + assert_eq!(assignments[r11_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r12_idx.0], Some(Allocation::Stack(1))); + assert_eq!(assignments[r13_idx.0], Some(Allocation::Reg(0))); + assert_eq!(assignments[r14_idx.0], Some(Allocation::Stack(2))); + assert_eq!(assignments[r15_idx.0], Some(Allocation::Reg(0))); + } + + #[test] + fn test_preferred_register_assignment_for_newborn_mov_source() { + let mut asm = Assembler::new(); + let block = asm.new_block(hir::BlockId(0), true, 0); + asm.set_current_block(block); + let label = asm.new_label("bb0"); + asm.write_label(label); + + let sp = NATIVE_STACK_PTR; + let new_sp = asm.add(sp, 0x20.into()); + asm.mov(sp, new_sp); + asm.cret(sp); + + asm.number_instructions(0); + let live_in = asm.analyze_liveness(); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + + let vreg_idx = new_sp.vreg_idx(); + assert_eq!(preferred_registers[vreg_idx.0], Some(sp.unwrap_reg())); + + let (assignments, num_stack_slots) = asm.linear_scan(intervals, 0, &preferred_registers); + assert_eq!(num_stack_slots, 0); + assert_eq!(assignments[vreg_idx.0], Some(Allocation::Fixed(sp.unwrap_reg()))); + } + + #[test] + fn test_debug_intervals() { + let TestFunc { mut asm, .. } = build_func(); + + // Number instructions + asm.number_instructions(16); + + // Get the debug output + let live_in = asm.analyze_liveness(); + let intervals = asm.build_intervals(live_in); + let output = debug_intervals(&asm, &intervals); + + // Verify it contains the grid structure + assert!(output.contains("v0")); // Header with vreg names + assert!(output.contains("---")); // Separator + assert!(output.contains("█")); // Live marker + assert!(output.contains(".")); // Dead marker + } + + #[test] + fn test_resolve_ssa() { + let TestFunc { mut asm, b1, b3, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers); + + asm.resolve_ssa(&intervals, &assignments); + + use crate::backend::current::ALLOC_REGS; + let regs = &ALLOC_REGS[..5]; + + // Edge b1->b2 (single succ): args=[UImm(1), v1], params=[v2, v3] + // v1->Reg(1), v2->Reg(1), v3->Reg(2) + // Reg copy: Reg(1)->Reg(2) -> Mov(regs[2], regs[1]) + // Imm move: Mov(regs[1], UImm(1)) + // Inserted in b1 before Jmp: [Label, Mov, Mov, Jmp] + let b1_insns = &asm.basic_blocks[b1.0].insns; + assert_eq!(b1_insns.len(), 4); + assert!(matches!(&b1_insns[1], Insn::Mov { dest, src } + if *dest == Opnd::Reg(regs[2]) && *src == Opnd::Reg(regs[1]))); + assert!(matches!(&b1_insns[2], Insn::Mov { dest, src } + if *dest == Opnd::Reg(regs[1]) && *src == Opnd::UImm(1))); + + // Edge b3->b2 (single succ): args=[v4, v5], params=[v2, v3] + // v4->Reg(3), v5->Reg(2), v2->Reg(1), v3->Reg(2) + // Reg copy: Reg(3)->Reg(1) -> Mov(regs[1], regs[3]) + // Reg(2)->Reg(2) is self-move, filtered + // Inserted in b3 before Jmp: [Label, Mul, Sub, Mov, Jmp] + let b3_insns = &asm.basic_blocks[b3.0].insns; + assert_eq!(b3_insns.len(), 5); + assert!(matches!(&b3_insns[3], Insn::Mov { dest, src } + if *dest == Opnd::Reg(regs[1]) && *src == Opnd::Reg(regs[3]))); + + // Verify original instructions in b3 are rewritten to physical registers. + // b3: Mul { left: r12, right: r13, out: r14 }, Sub { left: r13, right: UImm(1), out: r15 } + // r12->Reg(1), r13->Reg(2), r14->Reg(3), r15->Reg(2) + assert!(matches!(&b3_insns[1], Insn::Mul { left, right, out } + if *left == Opnd::Reg(regs[1]) && *right == Opnd::Reg(regs[2]) && *out == Opnd::Reg(regs[3]))); + assert!(matches!(&b3_insns[2], Insn::Sub { left, right, out } + if *left == Opnd::Reg(regs[2]) && *right == Opnd::UImm(1) && *out == Opnd::Reg(regs[2]))); + } + + #[test] + fn test_resolve_ssa_entry_params() { + let TestFunc { mut asm, b1, .. } = build_func(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers); + + // Entry block b1 has parameters [v0, v1]. + // With 5 registers: v0 -> Reg(0) = regs[0], arrival = C_ARG_OPNDS[0] = regs[0] -> self-move, filtered + // v1 -> Reg(1) = regs[1], arrival = C_ARG_OPNDS[1] = regs[1] -> self-move, filtered + // Before resolve_ssa, b1 has: [Label, Jmp] = 2 insns + assert_eq!(asm.basic_blocks[b1.0].insns.len(), 2); + + asm.resolve_ssa(&intervals, &assignments); + + // After resolve_ssa, b1 should still have the same number of insns + // (plus any edge moves, but no entry param moves since they're all self-moves). + // Edge b1->b2 inserts 2 moves before Jmp: [Label, Mov, Mov, Jmp] = 4 insns + // No additional entry param moves. + let b1_insns = &asm.basic_blocks[b1.0].insns; + assert_eq!(b1_insns.len(), 4); + // Verify the moves are edge moves (not entry param moves) + assert!(matches!(&b1_insns[1], Insn::Mov { .. })); + assert!(matches!(&b1_insns[2], Insn::Mov { .. })); + + // After resolve_ssa, edge args are cleared since the moves have been + // materialized as explicit Mov instructions. + if let Insn::Jmp(Target::Block(edge)) = &b1_insns[3] { + assert!(edge.args.is_empty(), "Edge args should be cleared after resolve_ssa"); + } else { + panic!("Expected Jmp at end of b1"); + } + } + + #[test] + fn test_resolve_ssa_entry_params_too_many_abort() { + let mut asm = Assembler::new(); + let block = asm.new_block(hir::BlockId(0), true, 0); + asm.set_current_block(block); + let label = asm.new_label("bb0"); + asm.write_label(label); + + for _ in 0..=C_ARG_OPNDS.len() { + let param = asm.new_vreg(64); + asm.basic_blocks[block.0].add_parameter(param); + } + asm.basic_blocks[block.0].push_insn(Insn::CRet(Opnd::UImm(0))); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(0); + let intervals = asm.build_intervals(live_in); + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers); + + asm.resolve_ssa(&intervals, &assignments); + + assert!(matches!(asm.basic_blocks[block.0].insns[1], Insn::Abort)); + } + + fn build_critical_edge() -> (Assembler, Opnd, Opnd, Opnd, Opnd, Opnd, BlockId, BlockId, BlockId) { + let mut asm = Assembler::new(); + + // Create blocks + let b1 = asm.new_block(hir::BlockId(0), true, 0); + let b2 = asm.new_block(hir::BlockId(1), false, 1); + let b3 = asm.new_block(hir::BlockId(2), false, 2); + + // b1: v0 = Add(123, 0), v1 = Add(v0, 456), Cmp(v1, 0), Jl(b2, [v0]), Jmp(b3, [v1]) + // v0 is live across b1->b2 edge AND v1 is live across b1->b3 edge + // This forces v0 and v1 to have overlapping live ranges -> different registers + asm.set_current_block(b1); + let label_b1 = asm.new_label("bb0"); + asm.write_label(label_b1); + let v0 = asm.new_vreg(64); + let v1 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: Opnd::UImm(123), right: Opnd::UImm(0), out: v0 }); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v0, right: Opnd::UImm(456), out: v1 }); + asm.basic_blocks[b1.0].push_insn(Insn::Cmp { left: v1, right: Opnd::UImm(0) }); + asm.basic_blocks[b1.0].push_insn(Insn::Jl(Target::Block(BranchEdge { target: b2, args: vec![v0] }))); + asm.basic_blocks[b1.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { target: b3, args: vec![v1] }))); + + // b2(v2): v3 = Add(v2, 789), Jmp(b3, [v3]) + asm.set_current_block(b2); + let label_b2 = asm.new_label("bb1"); + asm.write_label(label_b2); + let v2 = asm.new_block_param(64); + asm.basic_blocks[b2.0].add_parameter(v2); + let v3 = asm.new_vreg(64); + asm.basic_blocks[b2.0].push_insn(Insn::Add { left: v2, right: Opnd::UImm(789), out: v3 }); + asm.basic_blocks[b2.0].push_insn(Insn::Jmp(Target::Block(BranchEdge { target: b3, args: vec![v3] }))); + + // b3(v4): CRet(v4) + asm.set_current_block(b3); + let label_b3 = asm.new_label("bb2"); + asm.write_label(label_b3); + let v4 = asm.new_block_param(64); + asm.basic_blocks[b3.0].add_parameter(v4); + asm.basic_blocks[b3.0].push_insn(Insn::CRet(v4)); + + (asm, v0, v1, v2, v3, v4, b1, b2, b3) + } + + #[test] + fn test_resolve_critical_edge() { + let (mut asm, _v0, v1, _v2, v3, v4, b1, b2, b3) = build_critical_edge(); + + let live_in = asm.analyze_liveness(); + asm.number_instructions(16); + let intervals = asm.build_intervals(live_in); + let num_regs = 5; + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), num_regs, &preferred_registers); + + assert_eq!(asm.basic_blocks.len(), 3); + + // Verify v1 and v4 have different allocations (so moves are needed) + let v1_alloc = assignments[v1.vreg_idx().0].unwrap(); + let v4_alloc = assignments[v4.vreg_idx().0].unwrap(); + assert_ne!(v1_alloc, v4_alloc, "Test setup: v1 and v4 should have different allocations"); + + asm.resolve_ssa(&intervals, &assignments); + + // A new interstitial block should have been created for the critical edge b1->b3 + // b1->b3 is critical because b1 has 2 successors and b3 has 2 predecessors + assert_eq!(asm.basic_blocks.len(), 4); + let split_block_id = BlockId(3); + + // b1's Jmp should now target the split block instead of b3 + let b1_insns = &asm.basic_blocks[b1.0].insns; + let last_insn = b1_insns.last().unwrap(); + if let Insn::Jmp(Target::Block(edge)) = last_insn { + assert_eq!(edge.target, split_block_id); + } else { + panic!("Expected Jmp at end of b1"); + } + + // The split block should contain: Label, Mov(s), Jmp(b3) + let split_insns = &asm.basic_blocks[split_block_id.0].insns; + assert!(matches!(&split_insns[0], Insn::Label(_))); + let split_last = split_insns.last().unwrap(); + if let Insn::Jmp(Target::Block(edge)) = split_last { + assert_eq!(edge.target, b3); + assert!(edge.args.is_empty()); + } else { + panic!("Expected Jmp(b3) at end of split block"); + } + + // The split block should have a Mov for v1->v4 + let has_mov = split_insns.iter().any(|insn| matches!(insn, Insn::Mov { .. })); + assert!(has_mov, "Expected Mov in split block for v1->v4"); + + // b2->b3 is not a critical edge (b2 has single succ), so moves go before Jmp in b2 + let v3_alloc = assignments[v3.vreg_idx().0].unwrap(); + let b2_insns = &asm.basic_blocks[b2.0].insns; + if v3_alloc != v4_alloc { + // Check that a Mov was inserted before the Jmp in b2 + let second_last = &b2_insns[b2_insns.len() - 2]; + assert!(matches!(second_last, Insn::Mov { .. }), "Expected Mov before Jmp in b2"); + } + } + + #[test] + fn test_call() { + use crate::backend::current::ALLOC_REGS; + + let mut asm = Assembler::new(); + + // Single entry block + let b1 = asm.new_block(hir::BlockId(0), true, 0); + asm.set_current_block(b1); + let label = asm.new_label("bb0"); + asm.write_label(label); + + // v0 = param (entry block parameter) + let v0 = asm.new_block_param(64); + asm.basic_blocks[b1.0].add_parameter(v0); + + // v1 = Load(UImm(5)) + let v1 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Load { opnd: Opnd::UImm(5), out: v1 }); + + // v2 = Add(v1, UImm(1)) + let v2 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v1, right: Opnd::UImm(1), out: v2 }); + + // v3 = CCall { fptr: UImm(0xF00), opnds: [v2] } + let v3 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::CCall { + opnds: vec![v2], + fptr: Opnd::UImm(0xF00), + start_marker: None, + end_marker: None, + out: v3, + }); + + // v4 = Add(v3, v1) + let v4 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v3, right: v1, out: v4 }); + + // v5 = Add(v0, v4) + let v5 = asm.new_vreg(64); + asm.basic_blocks[b1.0].push_insn(Insn::Add { left: v0, right: v4, out: v5 }); + + // CRet(v5) + asm.basic_blocks[b1.0].push_insn(Insn::CRet(v5)); + + // Run liveness + numbering + intervals + linear scan with 2 registers + let live_in = asm.analyze_liveness(); + asm.number_instructions(0); + let intervals = asm.build_intervals(live_in); + let num_regs = 2; + let preferred_registers = asm.preferred_register_assignments(&intervals); + let (assignments, _) = asm.linear_scan(intervals.clone(), num_regs, &preferred_registers); + + let regs = &ALLOC_REGS[..num_regs]; + + // v0 should be spilled (long-lived, only 2 regs) + assert!(matches!(assignments[v0.vreg_idx().0], Some(Allocation::Stack(_))), + "v0 should be spilled to stack"); + // v1 should be in a register + assert!(matches!(assignments[v1.vreg_idx().0], Some(Allocation::Reg(_))), + "v1 should be in a register"); + + // Run the pipeline: handle_caller_saved_regs then resolve_ssa + asm.handle_caller_saved_regs(&intervals, &assignments, regs); + asm.resolve_ssa(&intervals, &assignments); + + let insns = &asm.basic_blocks[b1.0].insns; + + // Find CPush and CPopInto - they should be balanced. + let pushes: Vec<_> = insns.iter().filter(|i| matches!(i, Insn::CPushPair(..))).collect(); + let pops: Vec<_> = insns.iter().filter(|i| matches!(i, Insn::CPopPairInto(..))).collect(); + assert_eq!(pushes.len(), pops.len(), "CPush/CPopInto should be balanced"); + assert!(!pushes.is_empty(), "Expected at least one saved register across CCall"); + + // The survivor register should match v1's allocation + let v1_reg = match assignments[v1.vreg_idx().0].unwrap() { + Allocation::Reg(n) => Opnd::Reg(regs[n]), + Allocation::Fixed(reg) => Opnd::Reg(reg), + _ => unreachable!(), + }; + let pushed_v1 = pushes.iter().any(|insn| matches!(**insn, Insn::CPushPair(first, second) if first == v1_reg || second == v1_reg)); + let popped_v1 = pops.iter().any(|insn| matches!(**insn, Insn::CPopPairInto(first, second) if first == v1_reg || second == v1_reg)); + assert!(pushed_v1, "CPushPair should save v1's register"); + assert!(popped_v1, "CPopPairInto should restore v1's register"); + + // The CCall should have empty opnds and out = C_RET_OPND (rewritten to regs[0]) + let ccall = insns.iter().find(|i| matches!(i, Insn::CCall { .. })).unwrap(); + if let Insn::CCall { opnds, .. } = ccall { + assert!(opnds.is_empty(), "CCall opnds should be empty after handle_caller_saved_regs"); + } + + // v0 should be rewritten to a Stack operand + // Find an Add that uses a Stack operand (the v0+v4 add) + let has_stack_opnd = insns.iter().any(|i| { + if let Insn::Add { left: Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. }), .. } = i { + true + } else { + false + } + }); + assert!(has_stack_opnd, "v0 should be rewritten to a Stack memory operand"); + } +} diff --git a/zjit/src/backend/mod.rs b/zjit/src/backend/mod.rs index 4922421f18..f9a7e60a6b 100644 --- a/zjit/src/backend/mod.rs +++ b/zjit/src/backend/mod.rs @@ -1,3 +1,5 @@ +//! A multi-platform assembler generation backend. + #[cfg(target_arch = "x86_64")] pub mod x86_64; @@ -10,4 +12,8 @@ pub use x86_64 as current; #[cfg(target_arch = "aarch64")] pub use arm64 as current; +#[cfg(test)] +mod tests; + pub mod lir; +pub mod parcopy; diff --git a/zjit/src/backend/parcopy.rs b/zjit/src/backend/parcopy.rs new file mode 100644 index 0000000000..614bf68dcd --- /dev/null +++ b/zjit/src/backend/parcopy.rs @@ -0,0 +1,368 @@ +// This file came from here: https://github.com/bboissin/thesis_bboissin/blob/main/src/algorithm13.rs +// +// Copyright (c) 2025 bboissin +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// It's also Apache-2.0 licensed +use std::hash::Hash; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub struct Register(pub u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub struct RegisterCopy<T> { + pub source: T, + pub destination: T, +} + +// Algorithm 13 in Boissin's thesis: parallel copy sequentialization +// +// Takes a list of parallel copies, return a list of sequential copy operations +// such that each output register contains the same value as if the copies were +// parallel. +// The `spare` register may be used to break cycles and should not be contained +// in `parallel_copies`. The value of `spare` is undefined after the function +// returns. +// +// Varies slightly from the original algorithm as it splits the copies between +// pending and available to reduce state tracking. +pub fn sequentialize_register<T: PartialEq + Eq + Hash + Ord + std::fmt::Debug + Clone + Copy>(parallel_copies: &[RegisterCopy<T>], spare: T) -> Vec<RegisterCopy<T>> { + let mut sequentialized = Vec::new(); + // `resource` in the original code, this point to the current register + // holding a particular initial value. + // If a given Register is no longer needed, the value might be inaccurate. + let mut current_holder = std::collections::HashMap::new(); + // Copies that are pending, indexed by destination register. + // Use btree map to stay deterministic. + let mut pending = std::collections::BTreeMap::new(); + // If a copy can be materialized (nothing depends on the destination), we + // move it from pending into available. + let mut available = Vec::new(); + + for copy in parallel_copies { + if copy.source == spare || copy.destination == spare { + panic!("Spare register cannot be a source or destination of a copy"); + } + if let Some(_old_value) = pending.insert(copy.destination, copy) { + panic!( + "Destination register {:?} has multiple copies.", + copy.destination + ); + } + current_holder.insert(copy.source, copy.source); + } + for copy in parallel_copies { + // If we didn't record it, this means nothing depends on that register. + if !current_holder.contains_key(©.destination) { + pending.remove(©.destination); + available.push(copy); + } + } + while !pending.is_empty() || !available.is_empty() { + while let Some(copy) = available.pop() { + if let Some(source) = current_holder.get_mut(©.source) { + // Materialize the copy. + sequentialized.push(RegisterCopy { + source: source.clone(), + destination: copy.destination, + }); + if let Some(available_copy) = pending.remove(source) { + available.push(available_copy); + // Point to the new destination. + *source = copy.destination; + } else if *source == spare { + // Also point to new destination if we were copying from a + // spare, this lets us reuse spare for the next cycle. + *source = copy.destination; + } + } else { + panic!("No holder for source register {:?}", copy.source); + } + } + // If we have anything left, break the cycle by using the spare register + // on the first pending entry. + if let Some((destination, copy)) = pending.iter().next() { + sequentialized.push(RegisterCopy { + source: copy.destination, + destination: spare, + }); + current_holder.insert(copy.destination, spare); + available.push(copy); + let to_remove = *destination; + pending.remove(&to_remove); + } else { + // nothing pending. + break; + } + } + sequentialized +} + +#[cfg(test)] +mod tests { + use rand::RngExt; + use std::collections::HashMap; + + use super::*; + + // Assumes that each register initially contains the value matching its id. + fn execute_sequential(copies: &[RegisterCopy<Register>]) -> HashMap<Register, u32> { + let mut register_values = HashMap::new(); + // Initialize registers with their own ids as values. + for copy in copies { + register_values.insert(copy.source, copy.source.0); + } + for copy in copies { + let source_value = *register_values.get(©.source).unwrap(); + register_values.insert(copy.destination, source_value); + } + register_values + } + + fn execute_parallel(copies: &[RegisterCopy<Register>]) -> HashMap<Register, u32> { + let mut register_values = HashMap::new(); + // Initialize registers with their own ids as values. + for copy in copies { + register_values.insert(copy.source, copy.source.0); + } + // Execute copies. + for copy in copies { + register_values.insert(copy.destination, copy.source.0); + } + register_values + } + + #[test] + fn test_execute_sequential() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(3), + destination: Register(2), + }, + RegisterCopy { + source: Register(2), + destination: Register(4), + }, + RegisterCopy { + source: Register(2), + destination: Register(1), + }, + RegisterCopy { + source: Register(5), + destination: Register(3), + }, + ]; + let result = execute_sequential(&copies); + let expected: HashMap<Register, u32> = vec![ + (Register(1), 3), + (Register(2), 3), + (Register(3), 5), + (Register(4), 3), + (Register(5), 5), + ] + .into_iter() + .collect(); + assert_eq!(result, expected); + } + + #[test] + fn test_execute_sequential_2() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(4), + }, + RegisterCopy { + source: Register(3), + destination: Register(1), + }, + RegisterCopy { + source: Register(2), + destination: Register(3), + }, + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + ]; + let result = execute_sequential(&copies); + assert_eq!( + result, + Vec::from_iter([ + (Register(1), 3), + (Register(2), 3), + (Register(3), 2), + (Register(4), 1), + ]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_register_simple() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(2), + destination: Register(3), + }, + RegisterCopy { + source: Register(3), + destination: Register(4), + }, + ]; + + let spare = Register(5); + let result = sequentialize_register(&copies, spare); + let sequential_result = execute_sequential(&result); + assert_eq!( + sequential_result, + Vec::from_iter([ + (Register(1), 1), + (Register(2), 1), + (Register(3), 2), + (Register(4), 3), + ]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_cycle() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(2), + destination: Register(3), + }, + RegisterCopy { + source: Register(3), + destination: Register(1), + }, + ]; + let spare = Register(4); + let result = sequentialize_register(&copies, spare); + let mut sequential_result = execute_sequential(&result); + assert!(matches!(sequential_result.remove(&spare), Some(_))); + assert_eq!( + sequential_result, + Vec::from_iter([(Register(2), 1), (Register(3), 2), (Register(1), 3),]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_no_pending() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(3), + destination: Register(4), + }, + ]; + let spare = Register(5); + let result = sequentialize_register(&copies, spare); + let sequential_result = execute_sequential(&result); + assert_eq!( + sequential_result, + Vec::from_iter([ + (Register(1), 1), + (Register(2), 1), + (Register(3), 3), + (Register(4), 3), + ]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_with_fanin() { + let copies = vec![ + RegisterCopy { + source: Register(1), + destination: Register(2), + }, + RegisterCopy { + source: Register(1), + destination: Register(3), + }, + RegisterCopy { + source: Register(2), + destination: Register(1), + }, + ]; + let spare = Register(4); + let result = sequentialize_register(&copies, spare); + let sequential_result = execute_sequential(&result); + assert_eq!( + sequential_result, + Vec::from_iter([(Register(2), 1), (Register(3), 1), (Register(1), 2)]) + .into_iter() + .collect::<HashMap<_, _>>() + ); + } + + #[test] + fn test_sequentialize_rand() { + let mut rng = rand::rng(); + for _ in 0..1000 { + let num_copies = 100; + let mut copies = Vec::new(); + for i in 0..num_copies { + let dest = Register(i); + let src = Register(rng.random_range(0..num_copies)); + if src == dest { + continue; // Skip self-copies + } + copies.push(RegisterCopy { + source: src, + destination: dest, + }); + } + // shuffle the copies. + use rand::seq::SliceRandom; + + copies.shuffle(&mut rng); + let spare = Register(num_copies); + let result = sequentialize_register(&copies, spare); + let mut sequential_result = execute_sequential(&result); + // remove the spare register from the result. + sequential_result.remove(&spare); + assert_eq!(sequential_result, execute_parallel(&copies)); + } + } +} diff --git a/zjit/src/backend/tests.rs b/zjit/src/backend/tests.rs index aca775edd6..7174ac4c80 100644 --- a/zjit/src/backend/tests.rs +++ b/zjit/src/backend/tests.rs @@ -1,70 +1,22 @@ -#![cfg(test)] use crate::asm::CodeBlock; -use crate::backend::*; +use crate::backend::lir::*; use crate::cruby::*; -use crate::utils::c_callable; +use crate::codegen::c_callable; +use crate::options::rb_zjit_prepare_options; #[test] fn test_add() { - let mut asm = Assembler::new(0); + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); let out = asm.add(SP, Opnd::UImm(1)); let _ = asm.add(out, Opnd::UImm(2)); } -#[test] -fn test_alloc_regs() { - let mut asm = Assembler::new(0); - - // Get the first output that we're going to reuse later. - let out1 = asm.add(EC, Opnd::UImm(1)); - - // Pad some instructions in to make sure it can handle that. - let _ = asm.add(EC, Opnd::UImm(2)); - - // Get the second output we're going to reuse. - let out2 = asm.add(EC, Opnd::UImm(3)); - - // Pad another instruction. - let _ = asm.add(EC, Opnd::UImm(4)); - - // Reuse both the previously captured outputs. - let _ = asm.add(out1, out2); - - // Now get a third output to make sure that the pool has registers to - // allocate now that the previous ones have been returned. - let out3 = asm.add(EC, Opnd::UImm(5)); - let _ = asm.add(out3, Opnd::UImm(6)); - - // Here we're going to allocate the registers. - let result = asm.alloc_regs(Assembler::get_alloc_regs()); - - // Now we're going to verify that the out field has been appropriately - // updated for each of the instructions that needs it. - let regs = Assembler::get_alloc_regs(); - let reg0 = regs[0]; - let reg1 = regs[1]; - - match result.insns[0].out_opnd() { - Some(Opnd::Reg(value)) => assert_eq!(value, ®0), - val => panic!("Unexpected register value {:?}", val), - } - - match result.insns[2].out_opnd() { - Some(Opnd::Reg(value)) => assert_eq!(value, ®1), - val => panic!("Unexpected register value {:?}", val), - } - - match result.insns[5].out_opnd() { - Some(Opnd::Reg(value)) => assert_eq!(value, ®0), - val => panic!("Unexpected register value {:?}", val), - } -} - fn setup_asm() -> (Assembler, CodeBlock) { - return ( - Assembler::new(0), - CodeBlock::new_dummy(1024) - ); + rb_zjit_prepare_options(); // for get_option! on asm.compile + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy()) } // Test full codegen pipeline @@ -163,11 +115,10 @@ fn test_base_insn_out() ); // Load the pointer into a register - let ptr_reg = asm.load(Opnd::const_ptr(4351776248 as *const u8)); - let counter_opnd = Opnd::mem(64, ptr_reg, 0); + let ptr_opnd = Opnd::const_ptr(4351776248 as *const u8); // Increment and store the updated value - asm.incr_counter(counter_opnd, 1.into()); + asm.incr_counter(ptr_opnd, 1.into()); asm.compile_with_num_regs(&mut cb, 2); } @@ -189,17 +140,16 @@ fn test_c_call() // Make sure that the call's return value is usable asm.mov(Opnd::mem(64, SP, 0), ret_val); - asm.compile_with_num_regs(&mut cb, 1); + asm.compile(&mut cb).unwrap(); } #[test] fn test_alloc_ccall_regs() { - let mut asm = Assembler::new(0); - let out1 = asm.ccall(0 as *const u8, vec![]); - let out2 = asm.ccall(0 as *const u8, vec![out1]); + let (mut asm, mut cb) = setup_asm(); + let out1 = asm.ccall(std::ptr::null::<u8>(), vec![]); + let out2 = asm.ccall(std::ptr::null::<u8>(), vec![out1]); asm.mov(EC, out2); - let mut cb = CodeBlock::new_dummy(1024); - asm.compile_with_regs(&mut cb, None, Assembler::get_alloc_regs()); + asm.compile_with_regs(&mut cb, Assembler::get_alloc_regs()).unwrap(); } #[test] @@ -220,7 +170,7 @@ fn test_jcc_label() let label = asm.new_label("foo"); asm.cmp(EC, EC); - asm.je(label); + asm.push_insn(Insn::Je(label.clone())); asm.write_label(label); asm.compile_with_num_regs(&mut cb, 1); @@ -232,12 +182,12 @@ fn test_jcc_ptr() let (mut asm, mut cb) = setup_asm(); let side_exit = Target::CodePtr(cb.get_write_ptr().add_bytes(4)); - let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK)); + let not_mask = asm.not(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_MASK as i32)); asm.test( - Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG), + Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32), not_mask, ); - asm.jnz(side_exit); + asm.push_insn(Insn::Jnz(side_exit)); asm.compile_with_num_regs(&mut cb, 2); } @@ -266,7 +216,7 @@ fn test_jo() let arg0_untag = asm.sub(arg0, Opnd::Imm(1)); let out_val = asm.add(arg0_untag, arg1); - asm.jo(side_exit); + asm.push_insn(Insn::Jo(side_exit)); asm.mov(Opnd::mem(64, SP, 0), out_val); @@ -282,30 +232,10 @@ fn test_bake_string() { } #[test] -fn test_draining_iterator() { - let mut asm = Assembler::new(0); - - let _ = asm.load(Opnd::None); - asm.store(Opnd::None, Opnd::None); - let _ = asm.add(Opnd::None, Opnd::None); - - let mut iter = asm.into_draining_iter(); - - while let Some((index, insn)) = iter.next_unmapped() { - match index { - 0 => assert!(matches!(insn, Insn::Load { .. })), - 1 => assert!(matches!(insn, Insn::Store { .. })), - 2 => assert!(matches!(insn, Insn::Add { .. })), - _ => panic!("Unexpected instruction index"), - }; - } -} - -#[test] fn test_cmp_8_bit() { let (mut asm, mut cb) = setup_asm(); let reg = Assembler::get_alloc_regs()[0]; - asm.cmp(Opnd::Reg(reg).with_num_bits(8).unwrap(), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.cmp(Opnd::Reg(reg).with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); asm.compile_with_num_regs(&mut cb, 1); } @@ -314,7 +244,9 @@ fn test_cmp_8_bit() { fn test_no_pos_marker_callback_when_compile_fails() { // When compilation fails (e.g. when out of memory), the code written out is malformed. // We don't want to invoke the pos_marker callbacks with positions of malformed code. - let mut asm = Assembler::new(0); + let mut asm = Assembler::new(); + rb_zjit_prepare_options(); // for asm.compile + asm.new_block_without_id("test"); // Markers around code to exhaust memory limit let fail_if_called = |_code_ptr, _cb: &_| panic!("pos_marker callback should not be called"); @@ -324,6 +256,6 @@ fn test_no_pos_marker_callback_when_compile_fails() { asm.store(Opnd::mem(64, SP, 8), sum); asm.pos_marker(fail_if_called); - let cb = &mut CodeBlock::new_dummy(8); - assert!(asm.compile(cb, None).is_none(), "should fail due to tiny size limit"); + let cb = &mut CodeBlock::new_dummy_sized(8); + assert!(asm.compile(cb).is_err(), "should fail due to tiny size limit"); } diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index 2a02e1b725..d3bf847ab2 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1,15 +1,23 @@ -use std::mem::take; +use std::mem; use crate::asm::*; use crate::asm::x86_64::*; +use crate::codegen::split_patch_point; +use crate::stats::{CompileError, trace_compile_phase}; use crate::virtualmem::CodePtr; use crate::cruby::*; use crate::backend::lir::*; use crate::cast::*; +use crate::options::asm_dump; // Use the x86 register type for this platform pub type Reg = X86Reg; +/// Convert reg_no for MemBase::Reg into Reg, assuming it's a 64-bit GP register +pub fn mem_base_reg(reg_no: u8) -> Reg { + Reg { num_bits: 64, reg_type: RegType::GP, reg_no } +} + // Callee-saved registers pub const CFP: Opnd = Opnd::Reg(R13_REG); pub const EC: Opnd = Opnd::Reg(R12_REG); @@ -24,6 +32,7 @@ pub const C_ARG_OPNDS: [Opnd; 6] = [ Opnd::Reg(R8_REG), Opnd::Reg(R9_REG) ]; +pub const C_ARG_REGREGS: [Reg; 6] = [RDI_REG, RSI_REG, RDX_REG, RCX_REG, R8_REG, R9_REG]; // C return value register on this platform pub const C_RET_REG: Reg = RAX_REG; @@ -68,7 +77,7 @@ impl From<Opnd> for X86Opnd { "Attempted to lower an Opnd::None. This often happens when an out operand was not allocated for an instruction because the output of the instruction was not used. Please ensure you are using the output." ), - _ => panic!("unsupported x86 operand type") + _ => panic!("unsupported x86 operand type: {opnd:?}") } } } @@ -82,26 +91,40 @@ impl From<&Opnd> for X86Opnd { /// List of registers that can be used for register allocation. /// This has the same number of registers for x86_64 and arm64. -/// SCRATCH_REG is excluded. -pub const ALLOC_REGS: &'static [Reg] = &[ +/// SCRATCH0_OPND is excluded. +pub const ALLOC_REGS: &[Reg] = &[ RDI_REG, RSI_REG, RDX_REG, RCX_REG, R8_REG, R9_REG, - R10_REG, RAX_REG, ]; -impl Assembler -{ - /// Special scratch registers for intermediate processing. - /// This register is call-clobbered (so we don't have to save it before using it). - /// Avoid using if you can since this is used to lower [Insn] internally and - /// so conflicts are possible. - pub const SCRATCH_REG: Reg = R11_REG; - const SCRATCH0: X86Opnd = X86Opnd::Reg(Assembler::SCRATCH_REG); +/// Special scratch register for intermediate processing. It should be used only by +/// [`Assembler::x86_scratch_split`] or [`Assembler::new_with_scratch_reg`]. +const SCRATCH0_OPND: Opnd = Opnd::Reg(R11_REG); +const SCRATCH1_OPND: Opnd = Opnd::Reg(R10_REG); + +/// A scratch register available for use by resolve_ssa to break register copy cycles. +/// Must not overlap with ALLOC_REGS or other preserved registers. +pub const SCRATCH_REG: Reg = R11_REG; + +impl Assembler { + // This keeps frame growth below the +/-4096-byte displacement range we rely + // on for common stack-slot accesses on x86_64. + const MAX_FRAME_STACK_SLOTS: usize = 2048; + + /// Return an Assembler with scratch registers disabled in the backend, and a scratch register. + pub fn new_with_scratch_reg() -> (Self, Opnd) { + (Self::new_with_accept_scratch_reg(true), SCRATCH0_OPND) + } + + /// Return true if opnd contains a scratch reg + pub fn has_scratch_reg(opnd: Opnd) -> bool { + Self::has_reg(opnd, SCRATCH0_OPND.unwrap_reg()) + } /// Get the list of registers from which we can allocate on this platform pub fn get_alloc_regs() -> Vec<Reg> { @@ -119,102 +142,53 @@ impl Assembler } // These are the callee-saved registers in the x86-64 SysV ABI - // RBX, RSP, RBP, and R12–R15 + // RBX, RSP, RBP, and R12-R15 /// Split IR instructions for the x86 platform fn x86_split(mut self) -> Assembler { - let live_ranges: Vec<LiveRange> = take(&mut self.live_ranges); - let mut iterator = self.insns.into_iter().enumerate().peekable(); - let mut asm = Assembler::new_with_label_names(take(&mut self.label_names), live_ranges.len()); + let mut asm_local = Assembler::new_with_asm(&self); + let asm = &mut asm_local; + let mut iterator = self.instruction_iterator(); - while let Some((index, mut insn)) = iterator.next() { + while let Some((_index, mut insn)) = iterator.next(asm) { let is_load = matches!(insn, Insn::Load { .. } | Insn::LoadInto { .. }); - let mut opnd_iter = insn.opnd_iter_mut(); - - while let Some(opnd) = opnd_iter.next() { - // Lower Opnd::Value to Opnd::VReg or Opnd::UImm - match opnd { - Opnd::Value(value) if !is_load => { - // Since mov(mem64, imm32) sign extends, as_i64() makes sure - // we split when the extended value is different. - *opnd = if !value.special_const_p() || imm_num_bits(value.as_i64()) > 32 { - asm.load(*opnd) - } else { - Opnd::UImm(value.as_u64()) + let is_jump = insn.is_jump(); + + if !is_jump { + insn.for_each_operand_mut(|opnd| { + // Lower Opnd::Value to Opnd::VReg or Opnd::UImm + if let Opnd::Value(value) = opnd { + // If the value is a special constant, and it fits in 32 bits, + // then we're going to output this as just an immediate + if value.special_const_p() { + if imm_num_bits(value.as_i64()) > 32 { + *opnd = asm.load(*opnd); + } else { + *opnd = Opnd::UImm(value.as_u64()); + } + // If we're already loading it, don't load it again + // If it's a jump, then we want to let parallel move + // take care of the block params (otherwise we end up + // with loads between jump instructions) + } else if !is_load { + *opnd = asm.load(*opnd); } } - _ => {}, - }; + }); } - // When we split an operand, we can create a new VReg not in `live_ranges`. - // So when we see a VReg with out-of-range index, it's created from splitting - // from the loop above and we know it doesn't outlive the current instruction. - let vreg_outlives_insn = |vreg_idx| { - live_ranges - .get(vreg_idx) - .map_or(false, |live_range: &LiveRange| live_range.end() > index) - }; - // We are replacing instructions here so we know they are already // being used. It is okay not to use their output here. #[allow(unused_must_use)] match &mut insn { - Insn::Add { left, right, out } | - Insn::Sub { left, right, out } | - Insn::Mul { left, right, out } | - Insn::And { left, right, out } | - Insn::Or { left, right, out } | - Insn::Xor { left, right, out } => { - match (&left, &right, iterator.peek().map(|(_, insn)| insn)) { - // Merge this insn, e.g. `add REG, right -> out`, and `mov REG, out` if possible - (Opnd::Reg(_), Opnd::UImm(value), Some(Insn::Mov { dest, src })) - if out == src && left == dest && live_ranges[out.vreg_idx()].end() == index + 1 && uimm_num_bits(*value) <= 32 => { - *out = *dest; - asm.push_insn(insn); - iterator.next(); // Pop merged Insn::Mov - } - (Opnd::Reg(_), Opnd::Reg(_), Some(Insn::Mov { dest, src })) - if out == src && live_ranges[out.vreg_idx()].end() == index + 1 && *dest == *left => { - *out = *dest; - asm.push_insn(insn); - iterator.next(); // Pop merged Insn::Mov - } - _ => { - match (*left, *right) { - (Opnd::Mem(_), Opnd::Mem(_)) => { - *left = asm.load(*left); - *right = asm.load(*right); - }, - (Opnd::Mem(_), Opnd::UImm(_) | Opnd::Imm(_)) => { - *left = asm.load(*left); - }, - // Instruction output whose live range spans beyond this instruction - (Opnd::VReg { idx, .. }, _) => { - if vreg_outlives_insn(idx) { - *left = asm.load(*left); - } - }, - // We have to load memory operands to avoid corrupting them - (Opnd::Mem(_), _) => { - *left = asm.load(*left); - }, - // We have to load register operands to avoid corrupting them - (Opnd::Reg(_), _) => { - if *left != *out { - *left = asm.load(*left); - } - }, - // The first operand can't be an immediate value - (Opnd::UImm(_), _) => { - *left = asm.load(*left); - } - _ => {} - } - asm.push_insn(insn); - } - } + Insn::Add { .. } | + Insn::Sub { .. } | + Insn::Mul { .. } | + Insn::And { .. } | + Insn::Or { .. } | + Insn::Xor { .. } => { + asm.push_insn(insn); }, Insn::Cmp { left, right } => { // Replace `cmp REG, 0` (4 bytes) with `test REG, REG` (3 bytes) @@ -256,23 +230,11 @@ impl Assembler asm.push_insn(insn); }, // These instructions modify their input operand in-place, so we - // may need to load the input value to preserve it + // need to load the input value to preserve it Insn::LShift { opnd, .. } | Insn::RShift { opnd, .. } | Insn::URShift { opnd, .. } => { - match opnd { - // Instruction output whose live range spans beyond this instruction - Opnd::VReg { idx, .. } => { - if vreg_outlives_insn(*idx) { - *opnd = asm.load(*opnd); - } - }, - // We have to load non-reg operands to avoid corrupting them - Opnd::Mem(_) | Opnd::Reg(_) | Opnd::UImm(_) | Opnd::Imm(_) => { - *opnd = asm.load(*opnd); - }, - _ => {} - } + *opnd = asm.load(*opnd); asm.push_insn(insn); }, Insn::CSelZ { truthy, falsy, .. } | @@ -286,8 +248,8 @@ impl Assembler match *truthy { // If we have an instruction output whose live range // spans beyond this instruction, we have to load it. - Opnd::VReg { idx, .. } => { - if vreg_outlives_insn(idx) { + Opnd::VReg { idx: _, .. } => { + if true /* conservatively assume vreg outlives insn */ { *truthy = asm.load(*truthy); } }, @@ -321,8 +283,8 @@ impl Assembler match *opnd { // If we have an instruction output whose live range // spans beyond this instruction, we have to load it. - Opnd::VReg { idx, .. } => { - if vreg_outlives_insn(idx) { + Opnd::VReg { idx: _, .. } => { + if true /* conservatively assume vreg outlives insn */ { *opnd = asm.load(*opnd); } }, @@ -338,31 +300,11 @@ impl Assembler }, Insn::CCall { opnds, .. } => { assert!(opnds.len() <= C_ARG_OPNDS.len()); - - // Load each operand into the corresponding argument register. - if !opnds.is_empty() { - let mut args: Vec<(Reg, Opnd)> = vec![]; - for (idx, opnd) in opnds.into_iter().enumerate() { - args.push((C_ARG_OPNDS[idx].unwrap_reg(), *opnd)); - } - asm.parallel_mov(args); - } - - // Now we push the CCall without any arguments so that it - // just performs the call. - *opnds = vec![]; + // CCall argument setup is handled by handle_caller_saved_regs. asm.push_insn(insn); }, Insn::Lea { .. } => { - // Merge `lea` and `mov` into a single `lea` when possible - match (&insn, iterator.peek().map(|(_, insn)| insn)) { - (Insn::Lea { opnd, out }, Some(Insn::Mov { dest: Opnd::Reg(reg), src })) - if matches!(out, Opnd::VReg { .. }) && out == src && live_ranges[out.vreg_idx()].end() == index + 1 => { - asm.push_insn(Insn::Lea { opnd: *opnd, out: Opnd::Reg(*reg) }); - iterator.next(); // Pop merged Insn::Mov - } - _ => asm.push_insn(insn), - } + asm.push_insn(insn); }, _ => { asm.push_insn(insn); @@ -370,39 +312,372 @@ impl Assembler } } - asm + asm_local } - /// Emit platform-specific machine code - pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Option<Vec<CodePtr>> { + /// Split instructions using scratch registers. To maximize the use of the register pool + /// for VRegs, most splits should happen in [`Self::x86_split`]. However, some instructions + /// need to be split with registers after `alloc_regs`, e.g. for `compile_exits`, so + /// this splits them and uses scratch registers for it. + pub fn x86_scratch_split(self) -> Assembler { /// For some instructions, we want to be able to lower a 64-bit operand /// without requiring more registers to be available in the register - /// allocator. So we just use the SCRATCH0 register temporarily to hold + /// allocator. So we just use the SCRATCH0_OPND register temporarily to hold /// the value before we immediately use it. - fn emit_64bit_immediate(cb: &mut CodeBlock, opnd: &Opnd) -> X86Opnd { + fn split_64bit_immediate(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd) -> Opnd { match opnd { Opnd::Imm(value) => { // 32-bit values will be sign-extended - if imm_num_bits(*value) > 32 { - mov(cb, Assembler::SCRATCH0, opnd.into()); - Assembler::SCRATCH0 + if imm_num_bits(value) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd } else { - opnd.into() + opnd } }, Opnd::UImm(value) => { // 32-bit values will be sign-extended - if imm_num_bits(*value as i64) > 32 { - mov(cb, Assembler::SCRATCH0, opnd.into()); - Assembler::SCRATCH0 + if imm_num_bits(value as i64) > 32 { + asm.mov(scratch_opnd, opnd); + scratch_opnd } else { - imm_opnd(*value as i64) + Opnd::Imm(value as i64) } }, - _ => opnd.into() + _ => opnd + } + } + + /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg(NATIVE_BASE_PTR). + /// In general, `out` operand is a `VReg`, so it may use MemBase::Stack, but not MemBase::StackIndirect. + fn lower_stack_membase(opnd: Opnd, stack_state: &StackState) -> Opnd { + match opnd { + Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // Convert MemBase::Stack to MemBase::Reg(NATIVE_BASE_PTR) with the + // correct stack displacement. The stack slot value lives directly at + // [NATIVE_BASE_PTR + stack_disp], so we just adjust the base and + // combine displacements -- no indirection needed. + let Mem { base, disp: stack_disp, .. } = stack_state.stack_membase_to_mem(stack_membase); + Opnd::Mem(Mem { base, disp: stack_disp + opnd_disp, num_bits: opnd_num_bits }) + } + Opnd::Mem(Mem { base: MemBase::StackIndirect { .. }, .. }) => { + unreachable!("lower_stack_membase expects {opnd:?} to not have MemBase::StackIndirect") + } + _ => opnd, } } + /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg(NATIVE_BASE_PTR). + /// For MemBase::StackIndirect, load the pointer from the stack slot into a scratch register. + fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { + match opnd { + Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. }) => lower_stack_membase(opnd, stack_state), + Opnd::Mem(Mem { base: MemBase::StackIndirect { stack_idx }, disp: opnd_disp, num_bits: opnd_num_bits }) => { + // The spilled value (a pointer) lives at a stack slot. Load it + // into a scratch register, then use the register as the base. + let stack_mem = stack_state.stack_membase_to_mem(MemBase::Stack { stack_idx, num_bits: 64 }); + asm.load_into(scratch_opnd, Opnd::Mem(stack_mem)); + Opnd::Mem(Mem { + base: MemBase::Reg(scratch_opnd.unwrap_reg().reg_no), + disp: opnd_disp, + num_bits: opnd_num_bits, + }) + } + _ => opnd, + } + } + + /// If opnd is Opnd::Mem, set scratch_reg to *opnd. Return Some(Opnd::Mem) if it needs to be written back from scratch_reg. + fn split_memory_write(opnd: &mut Opnd, scratch_opnd: Opnd) -> Option<Opnd> { + if let Opnd::Mem(_) = opnd { + let mem_opnd = opnd.clone(); + *opnd = opnd.num_bits().map(|num_bits| scratch_opnd.with_num_bits(num_bits)).unwrap_or(scratch_opnd); + Some(mem_opnd) + } else { + None + } + } + + fn assert_out_is_phys_reg_or_stack_mem(out: Opnd) { + assert!( + matches!(out, Opnd::Reg(_) | Opnd::Mem(Mem { base: MemBase::Stack { .. }, .. })), + "x86_scratch_split expects out to be a physical register or stack memory, got {out:?}" + ); + } + + /// If both opnd and other are Opnd::Mem, split opnd with scratch_opnd. + fn split_if_both_memory(asm: &mut Assembler, opnd: Opnd, other: Opnd, scratch_opnd: Opnd) -> Opnd { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (opnd, other) { + asm.load_into(scratch_opnd.with_num_bits(opnd.rm_num_bits()), opnd); + scratch_opnd.with_num_bits(opnd.rm_num_bits()) + } else { + opnd + } + } + + /// Move src to dst, splitting it with scratch_opnd if it's a Mem-to-Mem move. Skip it if dst == src. + fn asm_mov(asm: &mut Assembler, dst: Opnd, src: Opnd, scratch_opnd: Opnd) { + if dst != src { + if let (Opnd::Mem(_), Opnd::Mem(_)) = (dst, src) { + asm.mov(scratch_opnd, src); + asm.mov(dst, scratch_opnd); + } else if let (Opnd::Mem(_), Opnd::Value(value)) = (dst, src) { + if imm_num_bits(value.as_i64()) > 32 { + asm.mov(scratch_opnd, src); + asm.mov(dst, scratch_opnd); + } else { + asm.mov(dst, src); + } + } else { + asm.mov(dst, src); + } + } + } + + // Prepare StackState to lower MemBase::Stack + let stack_state = StackState::new(self.stack_base_idx); + + let mut asm_local = Assembler::new(); + asm_local.accept_scratch_reg = true; + asm_local.stack_base_idx = self.stack_base_idx; + asm_local.label_names = self.label_names.clone(); + asm_local.num_vregs = self.num_vregs; + + // Create one giant block to linearize everything into + asm_local.new_block_without_id("linearized"); + + let asm = &mut asm_local; + + // Get linearized instructions with branch parameters expanded into ParallelMov + let linearized_insns = self.linearize_instructions(); + + for insn in linearized_insns.iter() { + let mut insn = insn.clone(); + match &mut insn { + Insn::Add { left, right, out } | + Insn::Sub { left, right, out } | + Insn::And { left, right, out } | + Insn::Or { left, right, out } | + Insn::Xor { left, right, out } => { + assert_out_is_phys_reg_or_stack_mem(*out); + + // Sequential pipeline to lower binops into x86 two-operand form. + // Uses SCRATCH0 for left, SCRATCH1 for right to avoid conflicts. + + // Phase 1: Protect right if it occupies the same location as out, + // since we'll overwrite out when moving left into it. + // Compare before lowering (Stack membases change during lowering). + let right_eq_out = out == right; + *out = lower_stack_membase(*out, &stack_state); + if right_eq_out { + asm.mov(SCRATCH1_OPND, *out); + *right = SCRATCH1_OPND; + } + + // Phase 2: Lower stack memory bases + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + if !right_eq_out { + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + } + + // Phase 3: If right is a Mem whose base register equals the out + // register, materialize it before we clobber out. + if let (Opnd::Reg(out_reg), Opnd::Mem(Mem { base: MemBase::Reg(base_reg_no), .. })) = (*out, *right) { + if out_reg.reg_no == base_reg_no { + asm.mov(SCRATCH1_OPND, *right); + *right = SCRATCH1_OPND; + } + } + + // Phase 4: x86 can't encode two memory operands. + if let (Opnd::Mem(_), Opnd::Mem(_)) = (*out, *right) { + asm.mov(SCRATCH1_OPND, *right); + *right = SCRATCH1_OPND; + } + + // Phase 5: Move left into out (x86 two-operand form: OP clobbers dst). + // For Mem out, split large left immediates first (mov [mem], imm64 is invalid). + if let Opnd::Mem(_) = *out { + *left = split_64bit_immediate(asm, *left, SCRATCH0_OPND); + } + asm_mov(asm, *out, *left, SCRATCH0_OPND); + *left = *out; + + // Phase 6: Split large right immediates (>32-bit) into a register. + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + + // Phase 7: Emit the instruction. + asm.push_insn(insn); + } + Insn::Mul { left, right, out } => { + assert_out_is_phys_reg_or_stack_mem(*out); + + *left = split_stack_membase(asm, *left, SCRATCH0_OPND, &stack_state); + *left = split_if_both_memory(asm, *left, *right, SCRATCH0_OPND); + *right = split_stack_membase(asm, *right, SCRATCH1_OPND, &stack_state); + *right = split_64bit_immediate(asm, *right, SCRATCH1_OPND); + *out = lower_stack_membase(*out, &stack_state); + + // imul doesn't have (Mem, Reg) encoding. Swap left and right in that case. + if let (Opnd::Mem(_), Opnd::Reg(_)) = (&left, &right) { + mem::swap(left, right); + } + + let (out, left) = (*out, *left); + asm.push_insn(insn); + asm_mov(asm, out, left, SCRATCH0_OPND); + } + &mut Insn::Not { opnd, out } | + &mut Insn::LShift { opnd, out, .. } | + &mut Insn::RShift { opnd, out, .. } | + &mut Insn::URShift { opnd, out, .. } => { + asm.push_insn(insn); + asm_mov(asm, out, opnd, SCRATCH0_OPND); + } + Insn::Test { left, right } | + Insn::Cmp { left, right } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + + let num_bits = match right { + Opnd::Imm(value) => Some(imm_num_bits(*value)), + Opnd::UImm(value) => Some(uimm_num_bits(*value)), + _ => None + }; + + // If the immediate is less than 64 bits (like 32, 16, 8), and the operand + // sizes match, then we can represent it as an immediate in the instruction + // without moving it to a register first. + // IOW, 64 bit immediates must always be moved to a register + // before comparisons, where other sizes may be encoded + // directly in the instruction. + let use_imm = num_bits.is_some() && left.num_bits() == num_bits && num_bits.unwrap() < 64; + if !use_imm { + *right = split_64bit_immediate(asm, *right, SCRATCH0_OPND); + } + asm.push_insn(insn); + } + // For compile_exits, support splitting simple C arguments here + Insn::CCall { opnds, .. } if !opnds.is_empty() => { + for (i, opnd) in opnds.iter().enumerate() { + asm.load_into(C_ARG_OPNDS[i], *opnd); + } + *opnds = vec![]; + asm.push_insn(insn); + } + Insn::CSelZ { truthy: left, falsy: right, out } | + Insn::CSelNZ { truthy: left, falsy: right, out } | + Insn::CSelE { truthy: left, falsy: right, out } | + Insn::CSelNE { truthy: left, falsy: right, out } | + Insn::CSelL { truthy: left, falsy: right, out } | + Insn::CSelLE { truthy: left, falsy: right, out } | + Insn::CSelG { truthy: left, falsy: right, out } | + Insn::CSelGE { truthy: left, falsy: right, out } => { + *left = split_stack_membase(asm, *left, SCRATCH1_OPND, &stack_state); + *right = split_stack_membase(asm, *right, SCRATCH0_OPND, &stack_state); + *right = split_if_both_memory(asm, *right, *left, SCRATCH0_OPND); + *out = lower_stack_membase(*out, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::Lea { opnd, out } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + *out = lower_stack_membase(*out, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND); + } + } + Insn::LeaJumpTarget { target, out } => { + if let Target::Label(_) = target { + asm.push_insn(Insn::LeaJumpTarget { out: SCRATCH0_OPND, target: target.clone() }); + asm.mov(*out, SCRATCH0_OPND); + } + } + Insn::Load { out, opnd } | + Insn::LoadInto { dest: out, opnd } => { + *opnd = split_stack_membase(asm, *opnd, SCRATCH0_OPND, &stack_state); + // Split stack membase on out before checking for memory write + *out = lower_stack_membase(*out, &stack_state); + let mem_out = split_memory_write(out, SCRATCH0_OPND); + asm.push_insn(insn); + if let Some(mem_out) = mem_out { + asm.store(mem_out, SCRATCH0_OPND.with_num_bits(mem_out.rm_num_bits())); + } + } + // Convert Opnd::const_ptr into Opnd::Mem. This split is done here to give + // a register for compile_exits. + &mut Insn::IncrCounter { mem, value } => { + assert!(matches!(mem, Opnd::UImm(_))); + asm.load_into(SCRATCH0_OPND, mem); + asm.incr_counter(Opnd::mem(64, SCRATCH0_OPND, 0), value); + } + &mut Insn::Mov { dest, src } => { + let dest = split_stack_membase(asm, dest, SCRATCH1_OPND, &stack_state); + let src = split_stack_membase(asm, src, SCRATCH0_OPND, &stack_state); + asm_mov(asm, dest, src, SCRATCH0_OPND); + } + // Handle various operand combinations for spills on compile_exits. + &mut Insn::Store { dest, src } => { + let num_bits = dest.rm_num_bits(); + let src = split_stack_membase(asm, src, SCRATCH0_OPND, &stack_state); + let dest = split_stack_membase(asm, dest, SCRATCH1_OPND, &stack_state); + + let src = match src { + Opnd::Reg(_) => src, + Opnd::Mem(_) => { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + Opnd::Imm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else if uimm_num_bits(imm as u64) <= num_bits { + // If the bit string is short enough for the destination, use the unsigned representation. + // Note that 64-bit and negative values are ruled out. + Opnd::UImm(imm as u64) + } else { + src + } + } + Opnd::UImm(imm) => { + // For 64 bit destinations, 32-bit values will be sign-extended + if num_bits == 64 && imm_num_bits(imm as i64) > 32 { + asm.mov(SCRATCH0_OPND, src); + SCRATCH0_OPND + } else { + src.into() + } + } + Opnd::Value(_) => { + asm.load_into(SCRATCH0_OPND, src); + SCRATCH0_OPND + } + src @ (Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during x86_scratch_split: {src:?}"), + }; + asm.store(dest, src); + } + &mut Insn::PatchPoint { ref target, invariant, version } => { + split_patch_point(asm, target, invariant, version); + } + _ => { + asm.push_insn(insn); + } + } + } + + asm_local + } + + /// Emit platform-specific machine code + pub fn x86_emit(&mut self, cb: &mut CodeBlock) -> Result<Vec<CodePtr>, CompileError> { fn emit_csel( cb: &mut CodeBlock, truthy: Opnd, @@ -412,21 +687,58 @@ impl Assembler cmov_neg: fn(&mut CodeBlock, X86Opnd, X86Opnd)){ // Assert that output is a register - out.unwrap_reg(); + let out_reg = out.unwrap_reg(); + + /// Check if a memory operand uses the given register as its base + fn mem_uses_reg(opnd: &Opnd, reg: Reg) -> bool { + if let Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) = opnd { + *reg_no == reg.reg_no + } else { + false + } + } + + /// Check if an operand aliases the given register (either as a + /// register operand or as a memory base). + fn aliases_reg(opnd: &Opnd, reg: Reg) -> bool { + match opnd { + Opnd::Reg(r) => r.reg_no == reg.reg_no, + Opnd::Mem(Mem { base: MemBase::Reg(reg_no), .. }) => *reg_no == reg.reg_no, + _ => false, + } + } // If the truthy value is a memory operand if let Opnd::Mem(_) = truthy { - if out != falsy { - mov(cb, out.into(), falsy.into()); + // If out aliases truthy, we must load truthy into the scratch + // register first to avoid clobbering it with the falsy mov. + if aliases_reg(&truthy, out_reg) { + mov(cb, SCRATCH0_OPND.into(), truthy.into()); + if out != falsy { + mov(cb, out.into(), falsy.into()); + } + cmov_fn(cb, out.into(), SCRATCH0_OPND.into()); + } else { + if out != falsy { + mov(cb, out.into(), falsy.into()); + } + cmov_fn(cb, out.into(), truthy.into()); } - - cmov_fn(cb, out.into(), truthy.into()); } else { - if out != truthy { - mov(cb, out.into(), truthy.into()); + // If out aliases falsy, we must load falsy into the scratch + // register first to avoid clobbering it with the truthy mov. + if aliases_reg(&falsy, out_reg) { + mov(cb, SCRATCH0_OPND.into(), falsy.into()); + if out != truthy { + mov(cb, out.into(), truthy.into()); + } + cmov_neg(cb, out.into(), SCRATCH0_OPND.into()); + } else { + if out != truthy { + mov(cb, out.into(), truthy.into()); + } + cmov_neg(cb, out.into(), falsy.into()); } - - cmov_neg(cb, out.into(), falsy.into()); } } @@ -449,7 +761,10 @@ impl Assembler // For each instruction let mut insn_idx: usize = 0; - while let Some(insn) = self.insns.get(insn_idx) { + assert_eq!(self.basic_blocks.len(), 1, "Assembler should be linearized into a single block before arm64_emit"); + let insns = &self.basic_blocks[0].insns; + + while let Some(insn) = insns.get(insn_idx) { match insn { Insn::Comment(text) => { cb.add_comment(text); @@ -505,33 +820,27 @@ impl Assembler } Insn::Add { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - add(cb, left.into(), opnd1); + add(cb, left.into(), right.into()); }, Insn::Sub { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - sub(cb, left.into(), opnd1); + sub(cb, left.into(), right.into()); }, Insn::Mul { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - imul(cb, left.into(), opnd1); + imul(cb, left.into(), right.into()); }, Insn::And { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - and(cb, left.into(), opnd1); + and(cb, left.into(), right.into()); }, Insn::Or { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - or(cb, left.into(), opnd1); + or(cb, left.into(), right.into()); }, Insn::Xor { left, right, .. } => { - let opnd1 = emit_64bit_immediate(cb, right); - xor(cb, left.into(), opnd1); + xor(cb, left.into(), right.into()); }, Insn::Not { opnd, .. } => { @@ -550,65 +859,6 @@ impl Assembler shr(cb, opnd.into(), shift.into()) }, - store_insn @ Insn::Store { dest, src } => { - let &Opnd::Mem(Mem { num_bits, base: MemBase::Reg(base_reg_no), disp: _ }) = dest else { - panic!("Unexpected Insn::Store destination in x64_emit: {dest:?}"); - }; - - // This kind of tricky clobber can only happen for explicit use of SCRATCH_REG, - // so we panic to get the author to change their code. - #[track_caller] - fn assert_no_clobber(store_insn: &Insn, user_use: u8, backend_use: Reg) { - assert_ne!( - backend_use.reg_no, - user_use, - "Emitting {store_insn:?} would clobber {user_use:?}, in conflict with its semantics" - ); - } - - let scratch = X86Opnd::Reg(Self::SCRATCH_REG); - let src = match src { - Opnd::Reg(_) => src.into(), - &Opnd::Mem(_) => { - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH_REG); - mov(cb, scratch, src.into()); - scratch - } - &Opnd::Imm(imm) => { - // For 64 bit destinations, 32-bit values will be sign-extended - if num_bits == 64 && imm_num_bits(imm) > 32 { - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH_REG); - mov(cb, scratch, src.into()); - scratch - } else if uimm_num_bits(imm as u64) <= num_bits { - // If the bit string is short enough for the destination, use the unsigned representation. - // Note that 64-bit and negative values are ruled out. - uimm_opnd(imm as u64) - } else { - src.into() - } - } - &Opnd::UImm(imm) => { - // For 64 bit destinations, 32-bit values will be sign-extended - if num_bits == 64 && imm_num_bits(imm as i64) > 32 { - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH_REG); - mov(cb, scratch, src.into()); - scratch - } else { - src.into() - } - } - &Opnd::Value(value) => { - assert_no_clobber(store_insn, base_reg_no, Self::SCRATCH_REG); - emit_load_gc_value(cb, &mut gc_offsets, scratch, value); - scratch - } - src @ (Opnd::None | Opnd::VReg { .. }) => panic!("Unexpected source operand during x86_emit: {src:?}") - - }; - mov(cb, dest.into(), src); - } - // This assumes only load instructions can contain references to GC'd Value operands Insn::Load { opnd, out } | Insn::LoadInto { dest: out, opnd } => { @@ -624,8 +874,7 @@ impl Assembler movsx(cb, out.into(), opnd.into()); }, - Insn::ParallelMov { .. } => unreachable!("{insn:?} should have been lowered at alloc_regs()"), - + Insn::Store { dest, src } | Insn::Mov { dest, src } => { mov(cb, dest.into(), src.into()); }, @@ -638,13 +887,13 @@ impl Assembler // Load address of jump target Insn::LeaJumpTarget { target, out } => { if let Target::Label(label) = target { + let out = *out; // Set output to the raw address of the label - cb.label_ref(*label, 7, |cb, src_addr, dst_addr| { + cb.label_ref(*label, 7, move |cb, src_addr, dst_addr| { let disp = dst_addr - src_addr; - lea(cb, Self::SCRATCH0, mem_opnd(8, RIP, disp.try_into().unwrap())); + lea(cb, out.into(), mem_opnd(8, RIP, disp.try_into().unwrap())); + Ok(()) }); - - mov(cb, out.into(), Self::SCRATCH0); } else { // Set output to the jump target's raw address let target_code = target.unwrap_code_ptr(); @@ -658,35 +907,32 @@ impl Assembler Insn::CPush(opnd) => { push(cb, opnd.into()); }, + Insn::CPushPair(opnd0, opnd1) => { + push(cb, opnd0.into()); + push(cb, opnd1.into()); + }, Insn::CPop { out } => { pop(cb, out.into()); }, Insn::CPopInto(opnd) => { pop(cb, opnd.into()); }, - - // Push and pop to the C stack all caller-save registers and the - // flags - Insn::CPushAll => { - let regs = Assembler::get_caller_save_regs(); - - for reg in regs { - push(cb, X86Opnd::Reg(reg)); - } - pushfq(cb); - }, - Insn::CPopAll => { - let regs = Assembler::get_caller_save_regs(); - - popfq(cb); - for reg in regs.into_iter().rev() { - pop(cb, X86Opnd::Reg(reg)); - } + Insn::CPopPairInto(opnd0, opnd1) => { + pop(cb, opnd0.into()); + pop(cb, opnd1.into()); }, // C function call Insn::CCall { fptr, .. } => { - call_ptr(cb, RAX, *fptr); + match fptr { + Opnd::UImm(fptr) => { + call_ptr(cb, RAX, *fptr as *const u8); + } + Opnd::Reg(_) => { + call(cb, fptr.into()); + } + _ => unreachable!("unsupported ccall fptr: {fptr:?}") + } }, Insn::CRet(opnd) => { @@ -700,30 +946,12 @@ impl Assembler // Compare Insn::Cmp { left, right } => { - let num_bits = match right { - Opnd::Imm(value) => Some(imm_num_bits(*value)), - Opnd::UImm(value) => Some(uimm_num_bits(*value)), - _ => None - }; - - // If the immediate is less than 64 bits (like 32, 16, 8), and the operand - // sizes match, then we can represent it as an immediate in the instruction - // without moving it to a register first. - // IOW, 64 bit immediates must always be moved to a register - // before comparisons, where other sizes may be encoded - // directly in the instruction. - if num_bits.is_some() && left.num_bits() == num_bits && num_bits.unwrap() < 64 { - cmp(cb, left.into(), right.into()); - } else { - let emitted = emit_64bit_immediate(cb, right); - cmp(cb, left.into(), emitted); - } + cmp(cb, left.into(), right.into()); } // Test and set flags Insn::Test { left, right } => { - let emitted = emit_64bit_immediate(cb, right); - test(cb, left.into(), emitted); + test(cb, left.into(), right.into()); } Insn::JmpOpnd(opnd) => { @@ -735,7 +963,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jmp_ptr(cb, code_ptr), Target::Label(label) => jmp_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jmp_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -743,7 +972,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => je_ptr(cb, code_ptr), Target::Label(label) => je_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => je_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -751,7 +981,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jne_ptr(cb, code_ptr), Target::Label(label) => jne_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jne_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -759,7 +990,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jl_ptr(cb, code_ptr), Target::Label(label) => jl_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jl_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -767,7 +999,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jg_ptr(cb, code_ptr), Target::Label(label) => jg_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jg_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -775,7 +1008,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jge_ptr(cb, code_ptr), Target::Label(label) => jge_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jge_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -783,7 +1017,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jbe_ptr(cb, code_ptr), Target::Label(label) => jbe_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jbe_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -791,7 +1026,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jb_ptr(cb, code_ptr), Target::Label(label) => jb_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jb_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } }, @@ -799,7 +1035,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jz_ptr(cb, code_ptr), Target::Label(label) => jz_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jz_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -807,7 +1044,8 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jnz_ptr(cb, code_ptr), Target::Label(label) => jnz_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jnz_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } @@ -816,13 +1054,14 @@ impl Assembler match *target { Target::CodePtr(code_ptr) => jo_ptr(cb, code_ptr), Target::Label(label) => jo_label(cb, label), - Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_side_exits"), + Target::Block(ref edge) => jo_label(cb, self.block_label(edge.target)), + Target::SideExit { .. } => unreachable!("Target::SideExit should have been compiled by compile_exits"), } } Insn::Joz(..) | Insn::Jonz(..) => unreachable!("Joz/Jonz should be unused for now"), - Insn::PatchPoint(_) | + Insn::PatchPoint { .. } => unreachable!("PatchPoint should have been lowered to PadPatchPoint in x86_scratch_split"), Insn::PadPatchPoint => { // If patch points are too close to each other or the end of the block, fill nop instructions if let Some(last_patch_pos) = last_patch_pos { @@ -843,6 +1082,7 @@ impl Assembler }, Insn::Breakpoint => int3(cb), + Insn::Abort => ud2(cb), Insn::CSelZ { truthy, falsy, out } => { emit_csel(cb, *truthy, *falsy, *out, cmovz, cmovnz); @@ -868,7 +1108,6 @@ impl Assembler Insn::CSelGE { truthy, falsy, out } => { emit_csel(cb, *truthy, *falsy, *out, cmovge, cmovl); } - Insn::LiveReg { .. } => (), // just a reg alloc signal, no code }; insn_idx += 1; @@ -876,55 +1115,311 @@ impl Assembler // Error if we couldn't write out everything if cb.has_dropped_bytes() { - return None + Err(CompileError::OutOfMemory) } else { // No bytes dropped, so the pos markers point to valid code for (insn_idx, pos) in pos_markers { - if let Insn::PosMarker(callback) = self.insns.get(insn_idx).unwrap() { - callback(pos, &cb); + if let Insn::PosMarker(callback) = insns.get(insn_idx).unwrap() { + callback(pos, cb); } else { panic!("non-PosMarker in pos_markers insn_idx={insn_idx} {self:?}"); } } - return Some(gc_offsets) + Ok(gc_offsets) } } /// Optimize and compile the stored instructions - pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Option<(CodePtr, Vec<CodePtr>)> { - let asm = self.x86_split(); - let mut asm = asm.alloc_regs(regs)?; - asm.compile_side_exits(); - - // Create label instances in the code block - for (idx, name) in asm.label_names.iter().enumerate() { - let label = cb.new_label(name.to_string()); - assert_eq!(label, Label(idx)); - } + pub fn compile_with_regs(self, cb: &mut CodeBlock, regs: Vec<Reg>) -> Result<(CodePtr, Vec<CodePtr>), CompileError> { + // The backend is allowed to use scratch registers only if it has not accepted them so far. + let use_scratch_regs = !self.accept_scratch_reg; + asm_dump!(self, init); - let start_ptr = cb.get_write_ptr(); - let gc_offsets = asm.x86_emit(cb); + let mut asm = trace_compile_phase("split", || self.x86_split()); - if let (Some(gc_offsets), false) = (gc_offsets, cb.has_dropped_bytes()) { - cb.link_labels(); + asm_dump!(asm, split); - Some((start_ptr, gc_offsets)) - } else { - cb.clear_labels(); + trace_compile_phase("regalloc", || { + trace_compile_phase("number_instructions", || asm.number_instructions(0)); + + let live_in = trace_compile_phase("analyze_liveness", || asm.analyze_liveness()); + let intervals = trace_compile_phase("build_intervals", || asm.build_intervals(live_in)); + + // Dump live intervals if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::live_intervals) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + } + } - None + let preferred_registers = trace_compile_phase("preferred_registers", || asm.preferred_register_assignments(&intervals)); + let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); + + let total_stack_slots = asm.stack_base_idx + num_stack_slots; + if total_stack_slots > Self::MAX_FRAME_STACK_SLOTS { + return Err(CompileError::NativeStackTooLarge); + } + + // Dump vreg-to-physical-register mapping if requested + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::alloc_regs) { + println!("LIR live_intervals:\n{}", crate::backend::lir::debug_intervals(&asm, &intervals)); + + println!("VReg assignments:"); + for (i, alloc) in assignments.iter().enumerate() { + if let Some(alloc) = alloc { + let range = &intervals[i].range; + let alloc_str = match alloc { + Allocation::Reg(n) => format!("{}", regs[*n]), + Allocation::Fixed(reg) => format!("{}", reg), + Allocation::Stack(n) => format!("Stack[{}]", n), + }; + println!(" v{} => {} (range: {:?}..{:?})", i, alloc_str, range.start, range.end); + } + } + } + } + + // Update FrameSetup slot_count to account for: + // 1) stack slots reserved for block params (stack_base_idx), and + // 2) register allocator spills (num_stack_slots). + trace_compile_phase("count_stack_slots", || { + for block in asm.basic_blocks.iter_mut() { + for insn in block.insns.iter_mut() { + if let Insn::FrameSetup { slot_count, .. } = insn { + *slot_count = total_stack_slots; + } + } + } + }); + + trace_compile_phase("resolve_ssa", || { + asm.handle_caller_saved_regs(&intervals, &assignments, &C_ARG_REGREGS); + asm.resolve_ssa(&intervals, &assignments); + }); + + Ok(()) + })?; + asm_dump!(asm, alloc_regs); + + // We are moved out of SSA after resolve_ssa + + // We put compile_exits after alloc_regs to avoid extending live ranges for VRegs spilled on side exits. + // Exit code is compiled into a separate list of instructions that we append + // to the last reachable block before scratch_split, so it gets linearized and split. + trace_compile_phase("compile_exits", || { + let exit_insns = asm.compile_exits(); + + // Append exit instructions to the last reachable block so they are + // included in linearize_instructions and processed by scratch_split. + if let Some(&last_block) = asm.block_order().last() { + for insn in exit_insns { + asm.basic_blocks[last_block.0].insns.push(insn); + asm.basic_blocks[last_block.0].insn_ids.push(None); + } + } + }); + asm_dump!(asm, compile_exits); + + if use_scratch_regs { + asm = trace_compile_phase("scratch_split", || asm.x86_scratch_split()); + asm_dump!(asm, scratch_split); + } else { + // For trampolines that use scratch registers, resolve ParallelMov without scratch_reg. + asm = trace_compile_phase("resolve_parallel_mov", || asm.resolve_parallel_mov_pass()); + asm_dump!(asm, resolve_parallel_mov); } + + trace_compile_phase("emit", || { + // Create label instances in the code block + for (idx, name) in asm.label_names.iter().enumerate() { + let label = cb.new_label(name.to_string()); + assert_eq!(label, Label(idx)); + } + + let start_ptr = cb.get_write_ptr(); + let gc_offsets = asm.x86_emit(cb).inspect_err(|_| cb.clear_labels())?; + assert!(!cb.has_dropped_bytes(), "emit should not drop bytes without error"); + + cb.link_labels().or(Err(CompileError::LabelLinkingFailure))?; + Ok((start_ptr, gc_offsets)) + }) } } #[cfg(test)] mod tests { - use crate::assertions::assert_disasm; + use insta::assert_snapshot; + use crate::assert_disasm_snapshot; + use crate::options::rb_zjit_prepare_options; use super::*; + const BOLD_BEGIN: &str = "\x1b[1m"; + const BOLD_END: &str = "\x1b[22m"; + fn setup_asm() -> (Assembler, CodeBlock) { - (Assembler::new(), CodeBlock::new_dummy()) + rb_zjit_prepare_options(); // for get_option! on asm.compile + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + (asm, CodeBlock::new_dummy()) + } + + fn stack_mem(stack_idx: usize) -> Opnd { + Opnd::Mem(Mem { + base: MemBase::Stack { stack_idx, num_bits: 64 }, + disp: 0, + num_bits: 64, + }) + } + + fn stack_indirect_mem(stack_idx: usize) -> Opnd { + Opnd::Mem(Mem { + base: MemBase::StackIndirect { stack_idx }, + disp: 0, + num_bits: 64, + }) + } + + #[derive(Clone, Copy, Debug)] + enum BinOpKind { + Add, + Sub, + And, + Or, + Xor, + } + + fn split_binop(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd) -> Assembler { + let (mut asm, _) = setup_asm(); + match kind { + BinOpKind::Add => asm.push_insn(Insn::Add { left, right, out }), + BinOpKind::Sub => asm.push_insn(Insn::Sub { left, right, out }), + BinOpKind::And => asm.push_insn(Insn::And { left, right, out }), + BinOpKind::Or => asm.push_insn(Insn::Or { left, right, out }), + BinOpKind::Xor => asm.push_insn(Insn::Xor { left, right, out }), + } + asm.x86_scratch_split() + } + + fn binop_mnemonic(kind: BinOpKind) -> &'static str { + match kind { + BinOpKind::Add => "add", + BinOpKind::Sub => "sub", + BinOpKind::And => "and", + BinOpKind::Or => "or", + BinOpKind::Xor => "xor", + } + } + + fn split_binop_disasm_lines(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd) -> Vec<String> { + let mut asm = split_binop(kind, left, right, out); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok(), "{kind:?}: x86_emit failed"); + + cb.disasm() + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(|line| { + line.split_once(": ") + .map(|(_, rest)| rest.to_string()) + .unwrap_or_else(|| line.to_string()) + }) + .collect() + } + + fn assert_split_binop_case(kind: BinOpKind, left: Opnd, right: Opnd, out: Opnd, case: &str) { + fn reg_names(reg: Reg) -> (&'static str, &'static str) { + const RAX_NO: u8 = RAX_REG.reg_no; + const RDI_NO: u8 = RDI_REG.reg_no; + const R13_NO: u8 = R13_REG.reg_no; + match reg.reg_no { + RAX_NO => ("rax", "eax"), + RDI_NO => ("rdi", "edi"), + R13_NO => ("r13", "r13d"), + _ => panic!("unexpected register in test helper: {reg:?}"), + } + } + + let lines = split_binop_disasm_lines(kind, left, right, out); + let mnemonic = binop_mnemonic(kind); + let matching: Vec<(usize, &String)> = lines.iter() + .enumerate() + .filter(|(_, line)| line.starts_with(&format!("{mnemonic} "))) + .collect(); + + assert_eq!(matching.len(), 1, "{kind:?} {case}: expected exactly one `{mnemonic}` in disasm, got {lines:?}"); + assert!( + !matching[0].1.contains("], qword ptr ["), + "{kind:?} {case}: emitted mem/mem `{mnemonic}` in disasm {lines:?}" + ); + + if let (Opnd::Reg(out_reg), Opnd::Imm(_) | Opnd::UImm(_)) = (out, left) { + let (out64, out32) = reg_names(out_reg); + let prelude = &lines[..matching[0].0]; + assert!( + prelude.iter().any(|line| + line.starts_with(&format!("mov {out64}, ")) || + line.starts_with(&format!("mov {out32}, ")) || + line.starts_with(&format!("movabs {out64}, ")) + ), + "{kind:?} {case}: expected left immediate to be materialized into output register before `{mnemonic}`, got {lines:?}" + ); + } + + if matches!(right, Opnd::Imm(value) if imm_num_bits(value) > 32) + || matches!(right, Opnd::UImm(value) if imm_num_bits(value as i64) > 32) + { + assert!(lines.iter().any(|line| line.starts_with("movabs ")), "{kind:?} {case}: expected movabs materialization for 64-bit immediate, got {lines:?}"); + } + } + + #[test] + fn test_lir_string() { + use crate::hir::SideExitReason; + + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + asm.stack_base_idx = 1; + + let label = asm.new_label("bb0"); + asm.write_label(label.clone()); + asm.push_insn(Insn::Comment("bb0(): foo@/tmp/a.rb:1".into())); + asm.frame_setup(JIT_PRESERVED_REGS); + + let val64 = asm.add(CFP, Opnd::UImm(64)); + asm.store(Opnd::mem(64, SP, 0x10), val64); + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; + asm.push_insn(Insn::Joz(val64, side_exit)); + asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); + asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); + + let val32 = asm.sub(Opnd::Value(Qtrue), Opnd::Imm(1)); + asm.store(Opnd::mem(64, EC, 0x10).with_num_bits(32), val32.with_num_bits(32)); + asm.push_insn(Insn::Je(label)); + asm.cret(val64); + + asm.frame_teardown(JIT_PRESERVED_REGS); + assert_disasm_snapshot!(lir_string(&mut asm), @" + test(): + bb0(): + # bb0(): foo@/tmp/a.rb:1 + FrameSetup 1, r13, rbx, r12 + v0 = Add r13, 0x40 + Store [rbx + 0x10], v0 + Joz Exit(Interrupt), v0 + Mov rdi, eax + Mov rsi, [rbx - 8] + v1 = Sub Value(0x14), Imm(1) + Store Mem32[r12 + 0x10], VReg32(v1) + Je bb0 + CRet v0 + FrameTeardown r13, rbx, r12 + "); } #[test] @@ -935,7 +1430,11 @@ mod tests { let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881c0ff000000"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: add rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c0ff000000"); } #[test] @@ -946,7 +1445,12 @@ mod tests { let _ = asm.add(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c01d8"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: add rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c01d8"); } #[test] @@ -957,7 +1461,11 @@ mod tests { let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881e0ff000000"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: and rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e0ff000000"); } #[test] @@ -968,7 +1476,12 @@ mod tests { let _ = asm.and(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c21d8"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: and rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c21d8"); } #[test] @@ -978,9 +1491,8 @@ mod tests { asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "4881f8ff000000", " - 0x0: cmp rax, 0xff - "); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"4881f8ff000000"); } #[test] @@ -990,10 +1502,11 @@ mod tests { asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "49bbffffffffffff00004c39d8", " - 0x0: movabs r11, 0xffffffffffff - 0xa: cmp rax, r11 + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); } #[test] @@ -1003,9 +1516,8 @@ mod tests { asm.cmp(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "4883f8ff", " - 0x0: cmp rax, -1 - "); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp rax, -1"); + assert_snapshot!(cb.hexdump(), @"4883f8ff"); } #[test] @@ -1017,7 +1529,8 @@ mod tests { asm.cmp(shape_opnd, Opnd::UImm(0xF000)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "6681780600f0"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp word ptr [rax + 6], 0xf000"); + assert_snapshot!(cb.hexdump(), @"6681780600f0"); } #[test] @@ -1029,7 +1542,8 @@ mod tests { asm.cmp(shape_opnd, Opnd::UImm(0xF000_0000)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "817804000000f0"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp dword ptr [rax + 4], 0xf0000000"); + assert_snapshot!(cb.hexdump(), @"817804000000f0"); } #[test] @@ -1040,7 +1554,11 @@ mod tests { let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881c8ff000000"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: or rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881c8ff000000"); } #[test] @@ -1051,7 +1569,12 @@ mod tests { let _ = asm.or(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c09d8"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: or rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c09d8"); } #[test] @@ -1062,7 +1585,11 @@ mod tests { let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881e8ff000000"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: sub rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881e8ff000000"); } #[test] @@ -1073,7 +1600,12 @@ mod tests { let _ = asm.sub(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c29d8"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: sub rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c29d8"); } #[test] @@ -1083,9 +1615,8 @@ mod tests { asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "48f7c0ff000000", " - 0x0: test rax, 0xff - "); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: test rax, 0xff"); + assert_snapshot!(cb.hexdump(), @"48f7c0ff000000"); } #[test] @@ -1095,7 +1626,11 @@ mod tests { asm.test(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 0); - assert_eq!(format!("{:x}", cb), "49bbffffffffffff00004c85d8"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0xffffffffffff + 0xa: test rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c85d8"); } #[test] @@ -1106,7 +1641,11 @@ mod tests { let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c04881f0ff000000"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: xor rax, 0xff + "); + assert_snapshot!(cb.hexdump(), @"4889c04881f0ff000000"); } #[test] @@ -1117,7 +1656,12 @@ mod tests { let _ = asm.xor(Opnd::Reg(RAX_REG), Opnd::UImm(0xFFFF_FFFF_FFFF)); asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4889c049bbffffffffffff00004c31d8"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, rax + 0x3: movabs r11, 0xffffffffffff + 0xd: xor rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"4889c049bbffffffffffff00004c31d8"); } #[test] @@ -1128,9 +1672,8 @@ mod tests { asm.mov(SP, sp); // should be merged to lea asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "488d5b08", {" - 0x0: lea rbx, [rbx + 8] - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: lea rbx, [rbx + 8]"); + assert_snapshot!(cb.hexdump(), @"488d5b08"); } #[test] @@ -1142,10 +1685,11 @@ mod tests { asm.mov(Opnd::mem(64, SP, 0), sp); // should NOT be merged to lea asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "488d4308488903", {" - 0x0: lea rax, [rbx + 8] - 0x4: mov qword ptr [rbx], rax - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: movabs r11, 0xffffffffffff + 0xa: cmp rax, r11 + "); + assert_snapshot!(cb.hexdump(), @"49bbffffffffffff00004c39d8"); } #[test] @@ -1159,7 +1703,15 @@ mod tests { asm.mov(Opnd::Reg(RAX_REG), result); asm.compile_with_num_regs(&mut cb, 2); - assert_eq!(format!("{:x}", cb), "488b43084885c0b814000000b900000000480f45c14889c0"); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov rax, qword ptr [rbx + 8] + 0x4: test rax, rax + 0x7: mov eax, 0x14 + 0xc: mov ecx, 0 + 0x11: cmovne rax, rcx + 0x15: mov rax, rax + "); + assert_snapshot!(cb.hexdump(), @"488b43084885c0b814000000b900000000480f45c14889c0"); } #[test] @@ -1170,9 +1722,8 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "4983c540", {" - 0x0: add r13, 0x40 - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); } #[test] @@ -1182,9 +1733,8 @@ mod tests { asm.add_into(CFP, Opnd::UImm(0x40)); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "4983c540", {" - 0x0: add r13, 0x40 - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: add r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983c540"); } #[test] @@ -1195,9 +1745,8 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "4983ed40", {" - 0x0: sub r13, 0x40 - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); } #[test] @@ -1207,9 +1756,8 @@ mod tests { asm.sub_into(CFP, Opnd::UImm(0x40)); asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "4983ed40", {" - 0x0: sub r13, 0x40 - "}); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: sub r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983ed40"); } #[test] @@ -1220,7 +1768,8 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983e540"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: and r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983e540"); } #[test] @@ -1231,7 +1780,8 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983cd40"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: or r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983cd40"); } #[test] @@ -1242,11 +1792,12 @@ mod tests { asm.mov(CFP, sp); // should be merged to add asm.compile_with_num_regs(&mut cb, 1); - assert_eq!(format!("{:x}", cb), "4983f540"); + assert_disasm_snapshot!(cb.disasm(), @" 0x0: xor r13, 0x40"); + assert_snapshot!(cb.hexdump(), @"4983f540"); } #[test] - fn test_reorder_c_args_no_cycle() { + fn test_ccall_resolve_parallel_moves_no_cycle() { crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); @@ -1256,14 +1807,15 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "b800000000ffd0", {" - 0x0: mov eax, 0 - 0x5: call rax - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov eax, 0 + 0x5: call rax + "); + assert_snapshot!(cb.hexdump(), @"b800000000ffd0"); } #[test] - fn test_reorder_c_args_single_cycle() { + fn test_ccall_resolve_parallel_moves_single_cycle() { crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); @@ -1275,17 +1827,18 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "4989f34889fe4c89dfb800000000ffd0", {" - 0x0: mov r11, rsi - 0x3: mov rsi, rdi - 0x6: mov rdi, r11 - 0x9: mov eax, 0 - 0xe: call rax - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rsi + 0x3: mov rsi, rdi + 0x6: mov rdi, r11 + 0x9: mov eax, 0 + 0xe: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989f34889fe4c89dfb800000000ffd0"); } #[test] - fn test_reorder_c_args_two_cycles() { + fn test_ccall_resolve_parallel_moves_two_cycles() { crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); @@ -1298,20 +1851,21 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "4989f34889fe4c89df4989cb4889d14c89dab800000000ffd0", {" - 0x0: mov r11, rsi - 0x3: mov rsi, rdi - 0x6: mov rdi, r11 - 0x9: mov r11, rcx - 0xc: mov rcx, rdx - 0xf: mov rdx, r11 - 0x12: mov eax, 0 - 0x17: call rax - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rcx + 0x3: mov rcx, rdx + 0x6: mov rdx, r11 + 0x9: mov r11, rsi + 0xc: mov rsi, rdi + 0xf: mov rdi, r11 + 0x12: mov eax, 0 + 0x17: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989cb4889d14c89da4989f34889fe4c89dfb800000000ffd0"); } #[test] - fn test_reorder_c_args_large_cycle() { + fn test_ccall_resolve_parallel_moves_large_cycle() { crate::options::rb_zjit_prepare_options(); let (mut asm, mut cb) = setup_asm(); @@ -1323,19 +1877,20 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); - assert_disasm!(cb, "4989f34889d64889fa4c89dfb800000000ffd0", {" - 0x0: mov r11, rsi - 0x3: mov rsi, rdx - 0x6: mov rdx, rdi - 0x9: mov rdi, r11 - 0xc: mov eax, 0 - 0x11: call rax - "}); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, rdx + 0x3: mov rdx, rdi + 0x6: mov rdi, rsi + 0x9: mov rsi, r11 + 0xc: mov eax, 0 + 0x11: call rax + "); + assert_snapshot!(cb.hexdump(), @"4989d34889fa4889f74c89deb800000000ffd0"); } #[test] #[ignore] - fn test_reorder_c_args_with_insn_out() { + fn test_ccall_resolve_parallel_moves_with_insn_out() { let (mut asm, mut cb) = setup_asm(); let rax = asm.load(Opnd::UImm(1)); @@ -1350,7 +1905,7 @@ mod tests { ]); asm.compile_with_num_regs(&mut cb, 3); - assert_disasm!(cb, "b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov eax, 1 0x5: mov ecx, 2 0xa: mov edx, 3 @@ -1361,7 +1916,123 @@ mod tests { 0x1b: mov rdx, r11 0x1e: mov eax, 0 0x23: call rax - "}); + "); + assert_snapshot!(cb.hexdump(), @"b801000000b902000000ba030000004889c74889ce4989cb4889d14c89dab800000000ffd0"); + } + + #[test] + fn test_ccall_register_preservation_even() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: push rdi + 0x15: push rsi + 0x16: push rdx + 0x17: push rcx + 0x18: mov eax, 0 + 0x1d: call rax + 0x1f: pop rcx + 0x20: pop rdx + 0x21: pop rsi + 0x22: pop rdi + 0x23: add rdi, rsi + 0x26: mov rdi, rdx + 0x29: add rdi, rcx + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000057565251b800000000ffd0595a5e5f4801f74889d74801cf"); + } + + #[test] + fn test_ccall_register_preservation_odd() { + let (mut asm, mut cb) = setup_asm(); + + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + let v2 = asm.load(3.into()); + let v3 = asm.load(4.into()); + let v4 = asm.load(5.into()); + asm.ccall(0 as _, vec![]); + _ = asm.add(v0, v1); + _ = asm.add(v2, v3); + _ = asm.add(v2, v4); + + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: mov edx, 3 + 0xf: mov ecx, 4 + 0x14: mov r8d, 5 + 0x1a: push rdi + 0x1b: push rsi + 0x1c: push rdx + 0x1d: push rcx + 0x1e: push r8 + 0x20: push 0 + 0x22: mov eax, 0 + 0x27: call rax + 0x29: pop r8 + 0x2b: pop r8 + 0x2d: pop rcx + 0x2e: pop rdx + 0x2f: pop rsi + 0x30: pop rdi + 0x31: add rdi, rsi + 0x34: mov rdi, rdx + 0x37: add rdi, rcx + 0x3a: mov rdi, rdx + 0x3d: add rdi, r8 + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be02000000ba03000000b90400000041b8050000005756525141506a00b800000000ffd041584158595a5e5f4801f74889d74801cf4889d74c01c7"); + } + + #[test] + fn test_cpush_pair() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpush_pair(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: push rdi + 0xb: push rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005756"); + } + + #[test] + fn test_cpop_pair_into() { + let (mut asm, mut cb) = setup_asm(); + let v0 = asm.load(1.into()); + let v1 = asm.load(2.into()); + asm.cpop_pair_into(v0, v1); + asm.compile_with_num_regs(&mut cb, ALLOC_REGS.len()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov edi, 1 + 0x5: mov esi, 2 + 0xa: pop rdi + 0xb: pop rsi + "); + assert_snapshot!(cb.hexdump(), @"bf01000000be020000005f5e"); } #[test] @@ -1378,12 +2049,13 @@ mod tests { asm.compile_with_num_regs(&mut cb, 1); - assert_disasm!(cb, "48837b1001bf04000000480f4f3b48893b", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: cmp qword ptr [rbx + 0x10], 1 0x5: mov edi, 4 0xa: cmovg rdi, qword ptr [rbx] 0xe: mov qword ptr [rbx], rdi - "}); + "); + assert_snapshot!(cb.hexdump(), @"48837b1001bf04000000480f4f3b48893b"); } #[test] @@ -1397,12 +2069,13 @@ mod tests { asm.compile_with_num_regs(&mut cb, 3); - assert_disasm!(cb, "48b830198dc8227f0000b904000000480f44c1488903", {" + assert_disasm_snapshot!(cb.disasm(), @" 0x0: movabs rax, 0x7f22c88d1930 0xa: mov ecx, 4 0xf: cmove rax, rcx 0x13: mov qword ptr [rbx], rax - "}); + "); + assert_snapshot!(cb.hexdump(), @"48b830198dc8227f0000b904000000480f44c1488903"); } #[test] @@ -1414,43 +2087,55 @@ mod tests { asm.mov(shape_opnd, Opnd::Imm(0x8000_0001)); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "c70001000080c70001000080", {" + + assert_disasm_snapshot!(cb.disasm(), @" 0x0: mov dword ptr [rax], 0x80000001 0x6: mov dword ptr [rax], 0x80000001 - "}); + "); + assert_snapshot!(cb.hexdump(), @"c70001000080c70001000080"); } #[test] - fn frame_setup_teardown() { + fn frame_setup_teardown_preserved_regs() { let (mut asm, mut cb) = setup_asm(); - asm.frame_setup(JIT_PRESERVED_REGS, 0); + asm.frame_setup(JIT_PRESERVED_REGS); asm.frame_teardown(JIT_PRESERVED_REGS); - asm.cret(C_RET_OPND); + asm.compile_with_num_regs(&mut cb, 0); - asm.frame_setup(&[], 5); - asm.frame_teardown(&[]); + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: push r13 + 0x6: push rbx + 0x7: push r12 + 0x9: sub rsp, 8 + 0xd: mov r13, qword ptr [rbp - 8] + 0x11: mov rbx, qword ptr [rbp - 0x10] + 0x15: mov r12, qword ptr [rbp - 0x18] + 0x19: mov rsp, rbp + 0x1c: pop rbp + 0x1d: ret + "); + assert_snapshot!(cb.hexdump(), @"554889e541555341544883ec084c8b6df8488b5df04c8b65e84889ec5dc3"); + } + #[test] + fn frame_setup_teardown_stack_base_idx() { + let (mut asm, mut cb) = setup_asm(); + asm.stack_base_idx = 5; + asm.frame_setup(&[]); + asm.frame_teardown(&[]); asm.compile_with_num_regs(&mut cb, 0); - assert_disasm!(cb, "554889e541555341544883ec084c8b6df8488b5df04c8b65e84889ec5dc3554889e54883ec304889ec5d", {" - 0x0: push rbp - 0x1: mov rbp, rsp - 0x4: push r13 - 0x6: push rbx - 0x7: push r12 - 0x9: sub rsp, 8 - 0xd: mov r13, qword ptr [rbp - 8] - 0x11: mov rbx, qword ptr [rbp - 0x10] - 0x15: mov r12, qword ptr [rbp - 0x18] - 0x19: mov rsp, rbp - 0x1c: pop rbp - 0x1d: ret - 0x1e: push rbp - 0x1f: mov rbp, rsp - 0x22: sub rsp, 0x30 - 0x26: mov rsp, rbp - 0x29: pop rbp - "}); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: push rbp + 0x1: mov rbp, rsp + 0x4: sub rsp, 0x30 + 0x8: mov rsp, rbp + 0xb: pop rbp + "); + assert_snapshot!(cb.hexdump(), @"554889e54883ec304889ec5d"); } #[test] @@ -1461,12 +2146,316 @@ mod tests { assert!(imitation_heap_value.heap_object_p()); asm.store(Opnd::mem(VALUE_BITS, SP, 0), imitation_heap_value.into()); + asm = asm.x86_scratch_split(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } let gc_offsets = asm.x86_emit(&mut cb).unwrap(); assert_eq!(1, gc_offsets.len(), "VALUE source operand should be reported as gc offset"); - assert_disasm!(cb, "49bb00100000000000004c891b", " + assert_disasm_snapshot!(cb.disasm(), @" 0x0: movabs r11, 0x1000 0xa: mov qword ptr [rbx], r11 "); + assert_snapshot!(cb.hexdump(), @"49bb00100000000000004c891b"); + } + + #[test] + fn test_csel_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let left = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let right = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 1, num_bits: 64 }, disp: 2, num_bits: 64 }); + let _ = asm.csel_e(left, right); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, qword ptr [rbp - 0xe] + 0x4: cmove r11, qword ptr [rbp - 8] + 0x9: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8b5df24c0f445df84c895df8"); + } + + #[test] + fn test_lea_split_memory_read() { + let (mut asm, mut cb) = setup_asm(); + + let opnd = Opnd::Mem(Mem { base: MemBase::Stack { stack_idx: 0, num_bits: 64 }, disp: 0, num_bits: 64 }); + let _ = asm.lea(opnd); + asm.compile_with_num_regs(&mut cb, 0); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: lea r11, [rbp - 8] + 0x4: mov qword ptr [rbp - 8], r11 + "); + assert_snapshot!(cb.hexdump(), @"4c8d5df84c895df8"); + } + + #[test] + fn test_add_split_direct_mem() { + // RAX is safe to be clobbered because it's an output + // c_ret <- add stack[0], stack[1] + let lines = split_binop_disasm_lines(BinOpKind::Add, stack_mem(0), stack_mem(1), C_RET_OPND); + + // load scratch0, [stack[0]] + // add scratch0, [stack[1]] + // mov c_ret, scratch0 + assert_snapshot!(lines.join("\n"), @" + mov rax, qword ptr [rbp - 8] + add rax, qword ptr [rbp - 0x10] + "); + } + + #[test] + fn test_add_split_stack_indirect_left_uses_scratch0_for_base_and_result() { + // stack[1] <- add stack[mem[0]], cfp + let lines = split_binop_disasm_lines(BinOpKind::Add, stack_indirect_mem(0), CFP, stack_mem(1)); + + // load scratch0, [stack[left_base]] + // load scratch0, [scratch0] + // seed [stack[out]] from scratch0, then add right in place + assert_snapshot!(lines.join("\n"), @" + mov r11, qword ptr [rbp - 8] + mov r11, qword ptr [r11] + mov qword ptr [rbp - 0x10], r11 + add qword ptr [rbp - 0x10], r13 + "); + } + + #[test] + fn test_add_split_stack_indirect_right_uses_separate_base_scratch() { + // mem[1] <- add cfp, mem[stack[0]] + let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_indirect_mem(0), stack_mem(1)); + + // load scratch1, [stack[right_base]] + // load scratch1, [scratch1] + // seed [stack[out]] from left, then add scratch1 in place + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [rbp - 8] + mov r10, qword ptr [r10] + mov qword ptr [rbp - 0x10], r13 + add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + fn test_add_split_two_stack_indirect_inputs_need_two_scratch_regs() { + // stack[2] <- add [stack[0]], [stack[1]] + let lines = split_binop_disasm_lines(BinOpKind::Add, + stack_indirect_mem(0), stack_indirect_mem(1), stack_mem(2)); + + // load scratch0, [stack[left_base]] + // load scratch0, [scratch0] + // load scratch1, [stack[right_base]] + // load scratch1, [scratch1] + // mov [stack[out]], scratch0; add [stack[out]], scratch1 + assert_snapshot!(lines.join("\n"), @" + mov r11, qword ptr [rbp - 8] + mov r10, qword ptr [rbp - 0x10] + mov r10, qword ptr [r10] + mov r11, qword ptr [r11] + mov qword ptr [rbp - 0x18], r11 + add qword ptr [rbp - 0x18], r10 + "); + } + + #[test] + fn test_add_split_memory_output_can_compute_in_place() { + // stack[1] <- add cfp, stack[0] + let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_mem(0), stack_mem(1)); + + // Seed the output slot with the left register, then perform the add in place. + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [rbp - 8] + mov qword ptr [rbp - 0x10], r13 + add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + fn test_add_split_reg_mem_mem_when_right_equals_out() { + // stack[1] <- add cfp, stack[1] + // + // Preserve the original RHS/output value first, then seed the output + // slot with the left register and finish the add in place. + let lines = split_binop_disasm_lines(BinOpKind::Add, CFP, stack_mem(1), stack_mem(1)); + + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [rbp - 0x10] + mov qword ptr [rbp - 0x10], r13 + add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + #[should_panic(expected = "x86_scratch_split expects out to be a physical register or stack memory")] + fn test_binop_split_rejects_non_stack_memory_output() { + let _ = split_binop(BinOpKind::Add, CFP, C_RET_OPND, Opnd::mem(64, CFP, 8)); + } + + #[test] + fn test_add_split_mem_mem_mem_when_right_equals_out() { + // stack[1] <- add stack[0], stack[1] + // + // Preserve the original RHS/output value first, then reload the LHS and + // write the result back to the output stack slot. + let mut asm = split_binop(BinOpKind::Add, stack_mem(0), stack_mem(1), stack_mem(1)); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r10, qword ptr [rbp - 0x10] + 0x4: mov r11, qword ptr [rbp - 8] + 0x8: mov qword ptr [rbp - 0x10], r11 + 0xc: add qword ptr [rbp - 0x10], r10 + "); + } + + #[test] + fn test_add_split_output_reg_reused_as_input_memory_base() { + // cfp <- add [cfp + 8], [cfp + 16] + // + // Preserve the RHS first because reusing `out` for the LHS would otherwise + // clobber the base register needed to address the RHS memory operand. + let mut asm = split_binop( + BinOpKind::Add, + Opnd::mem(64, CFP, 8), + Opnd::mem(64, CFP, 16), + CFP, + ); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r10, qword ptr [r13 + 0x10] + 0x4: mov r13, qword ptr [r13 + 8] + 0x8: add r13, r10 + "); + assert_snapshot!(cb.hexdump(), @"4d8b55104d8b6d084d01d5"); + } + + #[test] + fn test_add_split_output_reg_reused_as_stack_indirect_memory_base() { + // cfp <- add mem[stack[0]], mem[stack[1]] + // + // The LHS lowering reuses `out` as the temporary base register for the + // stack-indirect address, then the RHS uses the normal scratch register. + let mut asm = split_binop( + BinOpKind::Add, + stack_indirect_mem(0), + stack_indirect_mem(1), + CFP, + ); + let mut cb = CodeBlock::new_dummy(); + for name in &asm.label_names { + cb.new_label(name.to_string()); + } + assert!(asm.x86_emit(&mut cb).is_ok()); + + assert_disasm_snapshot!(cb.disasm(), @" + 0x0: mov r11, qword ptr [rbp - 8] + 0x4: mov r10, qword ptr [rbp - 0x10] + 0x8: mov r13, qword ptr [r11] + 0xb: add r13, qword ptr [r10] + "); + } + + #[test] + fn test_add_split_output_reg_reused_as_input_memory_base_with_imm() { + // cfp <- add 7, [cfp + 16] + // + // Preserve the RHS first because reusing `out` for the immediate + // materialization would otherwise clobber the base register needed for + // the memory operand. + let lines = split_binop_disasm_lines( + BinOpKind::Add, + Opnd::Imm(7), + Opnd::mem(64, CFP, 16), + CFP, + ); + + assert_snapshot!(lines.join("\n"), @" + mov r10, qword ptr [r13 + 0x10] + mov r13d, 7 + add r13, r10 + "); + } + + #[test] + fn test_binop_split_matrix() { + let left_cases = [ + ("reg", CFP), + ("mem_reg", Opnd::mem(64, C_RET_OPND, 8)), + ("mem_stack", stack_mem(0)), + ("mem_stack_indirect", stack_indirect_mem(0)), + ("imm32", Opnd::Imm(7)), + ("imm64", Opnd::UImm(0x1_0000_0000)), + ]; + let right_cases = [ + ("reg", C_RET_OPND), + ("mem_reg", Opnd::mem(64, CFP, 16)), + ("mem_stack", stack_mem(1)), + ("mem_stack_indirect", stack_indirect_mem(1)), + ("imm32", Opnd::Imm(9)), + ("imm64", Opnd::UImm(0x2_0000_0000)), + ]; + let out_cases = [ + ("out_reg", C_ARG_OPNDS[0]), + ("out_mem_stack", stack_mem(2)), + ]; + let alias_out_cases = [ + ("alias_out_cfp", CFP), + ("alias_out_reg", C_RET_OPND), + ("alias_out_mem_stack", stack_mem(1)), + ]; + let alias_mem_base_cases = [ + ( + "out_reused_as_left_mem_base", + Opnd::mem(64, CFP, 8), + C_RET_OPND, + CFP, + ), + ( + "out_reused_as_right_mem_base", + C_RET_OPND, + Opnd::mem(64, CFP, 16), + CFP, + ), + ( + "out_reused_as_both_mem_bases", + Opnd::mem(64, CFP, 8), + Opnd::mem(64, CFP, 16), + CFP, + ), + ]; + + for kind in [BinOpKind::Add, BinOpKind::Sub, BinOpKind::And, BinOpKind::Or, BinOpKind::Xor] { + for (left_name, left) in left_cases { + for (right_name, right) in right_cases { + for (out_name, out) in out_cases { + let case = format!("{left_name}/{right_name}/{out_name}"); + assert_split_binop_case(kind, left, right, out, &case); + } + } + } + + for (left_name, left) in left_cases { + for (alias_name, out) in alias_out_cases { + let case = format!("{left_name}/right_eq_out/{alias_name}"); + assert_split_binop_case(kind, left, out, out, &case); + } + } + + for (case, left, right, out) in alias_mem_base_cases { + assert_split_binop_case(kind, left, right, out, case); + } + } } } diff --git a/zjit/src/bitset.rs b/zjit/src/bitset.rs index 895bac8e33..986d537d9b 100644 --- a/zjit/src/bitset.rs +++ b/zjit/src/bitset.rs @@ -1,3 +1,5 @@ +//! Optimized bitset implementation. + type Entry = u128; const ENTRY_NUM_BITS: usize = Entry::BITS as usize; @@ -35,6 +37,16 @@ impl<T: Into<usize> + Copy> BitSet<T> { } } + /// Clear a bit. Returns whether the bit was previously set. + pub fn remove(&mut self, idx: T) -> bool { + debug_assert!(idx.into() < self.num_bits); + let entry_idx = idx.into() / ENTRY_NUM_BITS; + let bit_idx = idx.into() % ENTRY_NUM_BITS; + let was_set = (self.entries[entry_idx] & (1 << bit_idx)) != 0; + self.entries[entry_idx] &= !(1 << bit_idx); + was_set + } + pub fn get(&self, idx: T) -> bool { debug_assert!(idx.into() < self.num_bits); let entry_idx = idx.into() / ENTRY_NUM_BITS; @@ -55,6 +67,57 @@ impl<T: Into<usize> + Copy> BitSet<T> { } changed } + + /// Modify `self` to have bits set if they are set in either `self` or `other`. Returns true if `self` + /// was modified, and false otherwise. + /// `self` and `other` must have the same number of bits. + pub fn union_with(&mut self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + let mut changed = false; + for i in 0..self.entries.len() { + let before = self.entries[i]; + self.entries[i] |= other.entries[i]; + changed |= self.entries[i] != before; + } + changed + } + + /// Modify `self` to remove bits that are set in `other`. Returns true if `self` + /// was modified, and false otherwise. + /// `self` and `other` must have the same number of bits. + pub fn difference_with(&mut self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + let mut changed = false; + for i in 0..self.entries.len() { + let before = self.entries[i]; + self.entries[i] &= !other.entries[i]; + changed |= self.entries[i] != before; + } + changed + } + + /// Check if two BitSets are equal. + /// `self` and `other` must have the same number of bits. + pub fn equals(&self, other: &Self) -> bool { + assert_eq!(self.num_bits, other.num_bits); + self.entries == other.entries + } + + /// Returns an iterator over the indices of set bits. + /// Only iterates over bits that are set, not all possible indices. + pub fn iter_set_bits(&self) -> impl Iterator<Item = usize> + '_ { + self.entries.iter().enumerate().flat_map(move |(entry_idx, &entry)| { + let mut bits = entry; + std::iter::from_fn(move || { + if bits == 0 { + return None; + } + let bit_pos = bits.trailing_zeros() as usize; + bits &= bits - 1; // Clear the lowest set bit + Some(entry_idx * ENTRY_NUM_BITS + bit_pos) + }) + }).filter(move |&idx| idx < self.num_bits) + } } #[cfg(test)] @@ -65,40 +128,40 @@ mod tests { #[should_panic] fn get_over_capacity_panics() { let set = BitSet::with_capacity(0); - assert_eq!(set.get(0usize), false); + assert!(!set.get(0usize)); } #[test] fn with_capacity_defaults_to_zero() { let set = BitSet::with_capacity(4); - assert_eq!(set.get(0usize), false); - assert_eq!(set.get(1usize), false); - assert_eq!(set.get(2usize), false); - assert_eq!(set.get(3usize), false); + assert!(!set.get(0usize)); + assert!(!set.get(1usize)); + assert!(!set.get(2usize)); + assert!(!set.get(3usize)); } #[test] fn insert_sets_bit() { let mut set = BitSet::with_capacity(4); - assert_eq!(set.insert(1usize), true); - assert_eq!(set.get(1usize), true); + assert!(set.insert(1usize)); + assert!(set.get(1usize)); } #[test] fn insert_with_set_bit_returns_false() { let mut set = BitSet::with_capacity(4); - assert_eq!(set.insert(1usize), true); - assert_eq!(set.insert(1usize), false); + assert!(set.insert(1usize)); + assert!(!set.insert(1usize)); } #[test] fn insert_all_sets_all_bits() { let mut set = BitSet::with_capacity(4); set.insert_all(); - assert_eq!(set.get(0usize), true); - assert_eq!(set.get(1usize), true); - assert_eq!(set.get(2usize), true); - assert_eq!(set.get(3usize), true); + assert!(set.get(0usize)); + assert!(set.get(1usize)); + assert!(set.get(2usize)); + assert!(set.get(3usize)); } #[test] @@ -117,8 +180,46 @@ mod tests { right.insert(1usize); right.insert(2usize); left.intersect_with(&right); - assert_eq!(left.get(0usize), false); - assert_eq!(left.get(1usize), true); - assert_eq!(left.get(2usize), false); + assert!(!left.get(0usize)); + assert!(left.get(1usize)); + assert!(!left.get(2usize)); + } + + #[test] + fn test_iter_set_bits() { + let mut set: BitSet<usize> = BitSet::with_capacity(10); + set.insert(1usize); + set.insert(5usize); + set.insert(9usize); + + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![1, 5, 9]); + } + + #[test] + fn test_iter_set_bits_empty() { + let set: BitSet<usize> = BitSet::with_capacity(10); + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![]); + } + + #[test] + fn test_iter_set_bits_all() { + let mut set: BitSet<usize> = BitSet::with_capacity(5); + set.insert_all(); + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![0, 1, 2, 3, 4]); + } + + #[test] + fn test_iter_set_bits_large() { + let mut set: BitSet<usize> = BitSet::with_capacity(200); + set.insert(0usize); + set.insert(127usize); + set.insert(128usize); + set.insert(199usize); + + let set_bits: Vec<usize> = set.iter_set_bits().collect(); + assert_eq!(set_bits, vec![0, 127, 128, 199]); } } diff --git a/zjit/src/cast.rs b/zjit/src/cast.rs index bacc7245f3..52e2078cde 100644 --- a/zjit/src/cast.rs +++ b/zjit/src/cast.rs @@ -1,3 +1,7 @@ +//! Optimized [usize] casting trait. + +#![allow(clippy::wrong_self_convention)] + /// Trait for casting to [usize] that allows you to say `.as_usize()`. /// Implementation conditional on the cast preserving the numeric value on /// all inputs and being inexpensive. @@ -12,19 +16,19 @@ /// the method `into()` also causes a name conflict. pub(crate) trait IntoUsize { /// Convert to usize. Implementation conditional on width of [usize]. - fn as_usize(self) -> usize; + fn to_usize(self) -> usize; } #[cfg(target_pointer_width = "64")] impl IntoUsize for u64 { - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self as usize } } #[cfg(target_pointer_width = "64")] impl IntoUsize for u32 { - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self as usize } } @@ -32,7 +36,7 @@ impl IntoUsize for u32 { impl IntoUsize for u16 { /// Alias for `.into()`. For convenience so you could use the trait for /// all unsgined types. - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self.into() } } @@ -40,7 +44,7 @@ impl IntoUsize for u16 { impl IntoUsize for u8 { /// Alias for `.into()`. For convenience so you could use the trait for /// all unsgined types. - fn as_usize(self) -> usize { + fn to_usize(self) -> usize { self.into() } } diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 7ff5564d22..ee80ac0db5 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -1,25 +1,56 @@ +//! This module is for native code generation. + +#![allow(clippy::let_and_return)] + use std::cell::{Cell, RefCell}; use std::rc::Rc; use std::ffi::{c_int, c_long, c_void}; use std::slice; -use crate::asm::Label; -use crate::backend::current::{Reg, ALLOC_REGS}; -use crate::invariants::{track_bop_assumption, track_cme_assumption, track_single_ractor_assumption, track_stable_constant_names_assumption}; -use crate::gc::{append_gc_offsets, get_or_create_iseq_payload, get_or_create_iseq_payload_ptr, IseqStatus}; +use crate::backend::current::ALLOC_REGS; +use crate::invariants::{ + track_bop_assumption, track_cme_assumption, track_no_ep_escape_assumption, track_no_trace_point_assumption, + track_single_ractor_assumption, track_stable_constant_names_assumption, track_no_singleton_class_assumption, + track_root_box_assumption +}; +use crate::gc::append_gc_offsets; +use crate::payload::{IseqCodePtrs, IseqStatus, IseqVersion, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; use crate::state::ZJITState; -use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::compile_time_ns}; +use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; +use crate::stats::{counter_ptr, with_time_stat, trace_compile_phase, Counter, Counter::{compile_time_ns, exit_compile_error}}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; -use crate::backend::lir::{self, asm_comment, asm_ccall, Assembler, Opnd, Target, CFP, C_ARG_OPNDS, C_RET_OPND, EC, NATIVE_STACK_PTR, NATIVE_BASE_PTR, SCRATCH_OPND, SP}; -use crate::hir::{iseq_to_hir, Block, BlockId, BranchEdge, Invariant, RangeType, SideExitReason, SideExitReason::*, SpecialObjectType, SpecialBackrefSymbol, SELF_PARAM_IDX}; -use crate::hir::{Const, FrameState, Function, Insn, InsnId}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, Opnd, SP, SideExit, SideExitRecompile, Target, asm_ccall, asm_comment}; +use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; +use crate::hir::{BlockHandler, Const, FieldName, FrameState, Function, Insn, InsnId, Recompile, SendFallbackReason}; use crate::hir_type::{types, Type}; -use crate::options::get_option; +use crate::options::{get_option, PerfMap}; +use crate::cast::IntoUsize; + +/// The number of stack slots used for JITFrame. gen_save_pc_for_gc() writes +/// JITFrame into this number of slots at the bottom of the native stack. +const JIT_FRAME_SIZE: usize = 1; + +/// Default maximum number of compiled versions per ISEQ. +const DEFAULT_MAX_VERSIONS: usize = 2; + +/// Maximum number of compiled versions per ISEQ. +/// Configurable via --zjit-max-versions (default: 2). +pub fn max_iseq_versions() -> usize { + unsafe { crate::options::OPTIONS.as_ref() } + .map_or(DEFAULT_MAX_VERSIONS, |opts| opts.max_versions) +} + +/// Sentinel program counter stored in C frames when runtime checks are enabled. +const PC_POISON: Option<*const VALUE> = if cfg!(feature = "runtime_checks") { + Some(usize::MAX as *const VALUE) +} else { + None +}; /// Ephemeral code generation state struct JITState { - /// Instruction sequence for the method being compiled - iseq: IseqPtr, + /// ISEQ version that is being compiled, which will be used by PatchPoint + version: IseqVersionRef, /// Low-level IR Operands indexed by High-level IR's Instruction ID opnds: Vec<Option<Opnd>>, @@ -27,275 +58,492 @@ struct JITState { /// Labels for each basic block indexed by the BlockId labels: Vec<Option<Target>>, + /// JIT entry point for the `iseq` + jit_entries: Vec<Rc<RefCell<JITEntry>>>, + /// ISEQ calls that need to be compiled later - iseq_calls: Vec<Rc<RefCell<IseqCall>>>, + iseq_calls: Vec<IseqCallRef>, - /// The number of bytes allocated for basic block arguments spilled onto the C stack - c_stack_slots: usize, + /// The number of native stack slots reserved for JITFrame. + /// gen_save_pc_for_gc() writes JITFrame into the allocated space. + jit_frame_size: usize, } impl JITState { /// Create a new JITState instance - fn new(iseq: IseqPtr, num_insns: usize, num_blocks: usize, c_stack_slots: usize) -> Self { + fn new(version: IseqVersionRef, num_insns: usize, num_blocks: usize, jit_frame_size: usize) -> Self { JITState { - iseq, + version, opnds: vec![None; num_insns], labels: vec![None; num_blocks], + jit_entries: Vec::default(), iseq_calls: Vec::default(), - c_stack_slots, + jit_frame_size, } } /// Retrieve the output of a given instruction that has been compiled fn get_opnd(&self, insn_id: InsnId) -> lir::Opnd { - self.opnds[insn_id.0].expect(&format!("Failed to get_opnd({insn_id})")) + self.opnds[insn_id.0].unwrap_or_else(|| panic!("Failed to get_opnd({insn_id})")) + } + + /// Get the ISEQ for the version currently being compiled. + fn iseq(&self) -> IseqPtr { + unsafe { self.version.as_ref().iseq } } /// Find or create a label for a given BlockId - fn get_label(&mut self, asm: &mut Assembler, block_id: BlockId) -> Target { - match &self.labels[block_id.0] { + fn get_label(&mut self, asm: &mut Assembler, lir_block_id: lir::BlockId, hir_block_id: BlockId) -> Target { + // Extend labels vector if the requested index is out of bounds + if lir_block_id.0 >= self.labels.len() { + self.labels.resize(lir_block_id.0 + 1, None); + } + + match &self.labels[lir_block_id.0] { Some(label) => label.clone(), None => { - let label = asm.new_label(&format!("{block_id}")); - self.labels[block_id.0] = Some(label.clone()); + let label = asm.new_label(&format!("{hir_block_id}_{lir_block_id}")); + self.labels[lir_block_id.0] = Some(label.clone()); label } } } + } -/// CRuby API to compile a given ISEQ -#[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, _ec: EcPtr) -> *const u8 { - // Do not test the JIT code in HIR tests - if cfg!(test) { - return std::ptr::null(); +impl Assembler { + /// Emit a conditional jump that splits the current block, creating a new + /// fall-through block for instructions that follow. + fn split_block_jump(&mut self, jit: &mut JITState, emit: impl FnOnce(&mut Assembler, Target), target: Target) { + let hir_block_id = self.current_block().hir_block_id; + let rpo_idx = self.current_block().rpo_index; + + let fall_through_target = self.new_block(hir_block_id, false, rpo_idx); + let fall_through_edge = lir::BranchEdge { + target: fall_through_target, + args: vec![], + }; + emit(self, target); + self.jmp(Target::Block(fall_through_edge)); + + self.set_current_block(fall_through_target); + + let label = jit.get_label(self, fall_through_target, hir_block_id); + self.write_label(label); } +} - // Reject ISEQs with very large temp stacks. - // We cannot encode too large offsets to access locals in arm64. - let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) }; - if stack_max >= i8::MAX as u32 { - debug!("ISEQ stack too large: {stack_max}"); +macro_rules! define_split_jumps { + ($($name:ident => $insn:ident),+ $(,)?) => { + impl Assembler { + $( + fn $name(&mut self, jit: &mut JITState, target: Target) { + self.split_block_jump(jit, |asm, target| asm.push_insn(lir::Insn::$insn(target)), target); + } + )+ + } + }; +} + +define_split_jumps! { + jbe => Jbe, + je => Je, + jge => Jge, + jl => Jl, + jne => Jne, + jnz => Jnz, + jo => Jo, + jo_mul => JoMul, + jz => Jz, +} + +/// Record on the ISEQ payload whether `self` is guaranteed to be a heap object, +/// derived from the owning class of the method entry on `cfp`. Called from compile +/// triggers before the HIR is built so the `self`-producing instructions can be +/// typed precisely. Must be called while holding the VM lock (it writes the payload). +fn update_self_is_heap_object(iseq: IseqPtr, cfp: CfpPtr) { + let cme = unsafe { rb_vm_frame_method_entry(cfp) }; + let self_is_heap_object = !cme.is_null() + && iseq_self_is_heap_object(iseq, unsafe { (*cme).owner }); + get_or_create_iseq_payload(iseq).self_is_heap_object = self_is_heap_object; +} + +/// CRuby API to compile a given ISEQ. +/// If jit_exception is true, compile JIT code for handling exceptions. +/// See jit_compile_exception() for details. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_gen_entry_point(iseq: IseqPtr, ec: EcPtr, jit_exception: bool) -> *const u8 { + // Don't compile when there is insufficient native stack space + if unsafe { rb_ec_stack_check(ec as _) } != 0 { + incr_counter!(skipped_native_stack_full); return std::ptr::null(); } // Take a lock to avoid writing to ISEQ in parallel with Ractors. // with_vm_lock() does nothing if the program doesn't use Ractors. - let code_ptr = with_vm_lock(src_loc!(), || { - with_time_stat(compile_time_ns, || gen_iseq_entry_point(iseq)) - }); - - // Assert that the ISEQ compiles if RubyVM::ZJIT.assert_compiles is enabled - if ZJITState::assert_compiles_enabled() && code_ptr.is_null() { - let iseq_location = iseq_get_location(iseq, 0); - panic!("Failed to compile: {iseq_location}"); - } - - code_ptr -} + with_vm_lock(src_loc!(), || { + // The current frame is this ISEQ's method frame, so its method entry tells + // us the owning class and thus whether `self` is always a heap object. + update_self_is_heap_object(iseq, unsafe { get_ec_cfp(ec) }); + + let cb = ZJITState::get_code_block(); + let mut code_ptr = with_time_stat(compile_time_ns, || gen_iseq_entry_point(cb, iseq, jit_exception)); + + if let Err(err) = &code_ptr { + // Assert that the ISEQ compiles if RubyVM::ZJIT.assert_compiles is enabled. + // We assert only `jit_exception: false` cases until we support exception handlers. + if ZJITState::assert_compiles_enabled() && !jit_exception { + let iseq_location = iseq_get_location(iseq, 0); + panic!("Failed to compile: {iseq_location}: {err:?}"); + } -/// See [gen_iseq_entry_point_body]. This wrapper is to make sure cb.mark_all_executable() -/// is called even if gen_iseq_entry_point_body() partially fails and returns a null pointer. -fn gen_iseq_entry_point(iseq: IseqPtr) -> *const u8 { - let cb = ZJITState::get_code_block(); - let code_ptr = gen_iseq_entry_point_body(cb, iseq); + // For --zjit-stats, generate an entry that just increments exit_compilation_failure and exits + if get_option!(stats) { + code_ptr = gen_compile_error_counter(cb, err); + } + } - // Always mark the code region executable if asm.compile() has been used. - // We need to do this even if code_ptr is null because, whether gen_entry() or - // gen_function_stub() fails or not, gen_function() has already used asm.compile(). - cb.mark_all_executable(); + // Always mark the code region executable if asm.compile() has been used. + // We need to do this even if code_ptr is None because gen_iseq() may have already used asm.compile(). + cb.mark_all_executable(); - code_ptr.map_or(std::ptr::null(), |ptr| ptr.raw_ptr(cb)) + code_ptr.map_or(std::ptr::null(), |ptr| ptr.raw_ptr(cb)) + }) } /// Compile an entry point for a given ISEQ -fn gen_iseq_entry_point_body(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<CodePtr> { - // Compile ISEQ into High-level IR - let function = compile_iseq(iseq)?; +fn gen_iseq_entry_point(cb: &mut CodeBlock, iseq: IseqPtr, jit_exception: bool) -> Result<CodePtr, CompileError> { + // We don't support exception handlers yet + if jit_exception { + return Err(CompileError::ExceptionHandler); + } - // Compile the High-level IR - let Some((start_ptr, gc_offsets, jit)) = gen_function(cb, iseq, &function) else { - debug!("Failed to compile iseq: gen_function failed: {}", iseq_get_location(iseq, 0)); - return None; - }; + let iseq_name = iseq_get_location(iseq, 0); + trace_compile_phase(&iseq_name, || { + // Compile ISEQ into High-level IR + let function = crate::stats::with_time_stat(Counter::compile_hir_time_ns, || compile_iseq(iseq).inspect_err(|_| { + incr_counter!(failed_iseq_count); + }))?; - // Compile an entry point to the JIT code - let Some(entry_ptr) = gen_entry(cb, iseq, &function, start_ptr) else { - debug!("Failed to compile iseq: gen_entry failed: {}", iseq_get_location(iseq, 0)); - return None; - }; + // Compile the High-level IR + let IseqCodePtrs { start_ptr, .. } = gen_iseq(cb, iseq, Some(&function)).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq, 0)); + })?; - // Stub callee ISEQs for JIT-to-JIT calls - for iseq_call in jit.iseq_calls.iter() { - gen_iseq_call(cb, iseq, iseq_call)?; - } + Ok(start_ptr) + }) +} - // Remember the block address to reuse it later +/// Invalidate an ISEQ version and allow it to be recompiled on the next call. +/// Both PatchPoint invalidation and exit-profiling recompilation go through this +/// function, serving as the central point for all invalidation/recompile decisions. +/// +/// TODO: evolve this into a general `handle_event(iseq, event)` state machine that +/// handles all compile lifecycle events (interpreter profiles, JIT profiles, invalidation, +/// GC) so that all compile/recompile tuning decisions live in one place. +pub fn invalidate_iseq_version(cb: &mut CodeBlock, iseq: IseqPtr, version: &mut IseqVersionRef) { let payload = get_or_create_iseq_payload(iseq); - payload.status = IseqStatus::Compiled(start_ptr); - payload.iseq_calls.extend(jit.iseq_calls); - append_gc_offsets(iseq, &gc_offsets); - - // Return a JIT code address - Some(entry_ptr) + if !unsafe { version.as_ref() }.is_invalidated() + && payload.versions.len() < max_iseq_versions() + { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + unsafe { rb_iseq_reset_jit_func(iseq) }; + + // Recompile JIT-to-JIT calls into the invalidated ISEQ + for incoming in unsafe { version.as_ref() }.incoming.iter() { + if let Err(err) = gen_iseq_call(cb, incoming) { + debug!("{err:?}: gen_iseq_call failed during invalidation: {}", iseq_get_location(incoming.iseq.get(), 0)); + } + } + } } /// Stub a branch for a JIT-to-JIT call -fn gen_iseq_call(cb: &mut CodeBlock, caller_iseq: IseqPtr, iseq_call: &Rc<RefCell<IseqCall>>) -> Option<()> { - // Compile a function stub - let Some(stub_ptr) = gen_function_stub(cb, iseq_call.clone()) else { - // Failed to compile the stub. Bail out of compiling the caller ISEQ. - debug!("Failed to compile iseq: could not compile stub: {} -> {}", - iseq_get_location(caller_iseq, 0), iseq_get_location(iseq_call.borrow().iseq, 0)); - return None; - }; - - // Update the JIT-to-JIT call to call the stub - let stub_addr = stub_ptr.raw_ptr(cb); - let iseq = iseq_call.borrow().iseq; - iseq_call.borrow_mut().regenerate(cb, |asm| { - asm_comment!(asm, "call function stub: {}", iseq_get_location(iseq, 0)); - asm.ccall(stub_addr, vec![]); - }); - Some(()) +pub fn gen_iseq_call(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<(), CompileError> { + trace_compile_phase("compile_stub", || { + // Compile a function stub + let stub_ptr = gen_function_stub(cb, iseq_call.clone()).inspect_err(|err| { + debug!("{err:?}: gen_function_stub failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; + + // Update the JIT-to-JIT call to call the stub + let stub_addr = stub_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call function stub: {}", iseq_get_location(iseq, 0)); + asm.ccall_into(C_RET_OPND, stub_addr, vec![]); + }); + Ok(()) + }) } /// Write an entry to the perf map in /tmp -fn register_with_perf(iseq_name: String, start_ptr: usize, code_size: usize) { +fn register_with_perf(symbol_name: String, start_ptr: usize, code_size: usize) { use std::io::Write; let perf_map = format!("/tmp/perf-{}.map", std::process::id()); - let Ok(mut file) = std::fs::OpenOptions::new().create(true).append(true).open(&perf_map) else { + let Ok(file) = std::fs::OpenOptions::new().create(true).append(true).open(&perf_map) else { debug!("Failed to open perf map file: {perf_map}"); return; }; - let Ok(_) = writeln!(file, "{:#x} {:#x} zjit::{}", start_ptr, code_size, iseq_name) else { - debug!("Failed to write {iseq_name} to perf map file: {perf_map}"); + let mut file = std::io::BufWriter::new(file); + let Ok(_) = writeln!(file, "{start_ptr:#x} {code_size:#x} ZJIT: {symbol_name}") else { + debug!("Failed to write {symbol_name} to perf map file: {perf_map}"); return; }; } -/// Compile a JIT entry -fn gen_entry(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function, function_ptr: CodePtr) -> Option<CodePtr> { +/// Compile a shared JIT entry trampoline +pub fn gen_entry_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { // Set up registers for CFP, EC, SP, and basic block arguments let mut asm = Assembler::new(); - gen_entry_prologue(&mut asm, iseq); - gen_entry_params(&mut asm, iseq, function.block(BlockId(0))); + asm.new_block_without_id("gen_entry_trampoline"); + gen_entry_prologue(&mut asm); - // Jump to the first block using a call instruction - asm.ccall(function_ptr.raw_ptr(cb) as *const u8, vec![]); + // Jump to the first block using a call instruction. This trampoline is used + // as rb_zjit_func_t in jit_exec(), which takes (EC, CFP, rb_jit_func_t). + // So C_ARG_OPNDS[2] is rb_jit_func_t, which is (EC, CFP) -> VALUE. + let out = asm.ccall_reg(C_ARG_OPNDS[2], VALUE_BITS); // Restore registers for CFP, EC, and SP after use asm_comment!(asm, "return to the interpreter"); asm.frame_teardown(lir::JIT_PRESERVED_REGS); - asm.cret(C_RET_OPND); + asm.cret(out); + + let (code_ptr, gc_offsets) = asm.compile(cb)?; + assert!(gc_offsets.is_empty()); + if get_option!(perf).is_some() { + let start_ptr = code_ptr.raw_addr(cb); + let end_ptr = cb.get_write_ptr().raw_addr(cb); + let code_size = end_ptr - start_ptr; + register_with_perf("entry trampoline".into(), start_ptr, code_size); + } + Ok(code_ptr) +} - if get_option!(dump_lir) { - println!("LIR:\nJIT entry for {}:\n{:?}", iseq_name(iseq), asm); +/// Compile an ISEQ into machine code if not compiled yet +fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { + // Return an existing pointer if it's already compiled + let payload = get_or_create_iseq_payload(iseq); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + match last_status { + Some(IseqStatus::Compiled(code_ptrs)) => return Ok(code_ptrs.clone()), + Some(IseqStatus::CantCompile(err)) => return Err(err.clone()), + _ => {}, + } + // If the ISEQ already has max versions, do not compile a new version. + if payload.versions.len() >= max_iseq_versions() { + return Err(CompileError::IseqVersionLimitReached); } - let result = asm.compile(cb).map(|(start_ptr, _)| start_ptr); - if let Some(start_addr) = result { - if get_option!(perf) { - let start_ptr = start_addr.raw_ptr(cb) as usize; - let end_ptr = cb.get_write_ptr().raw_ptr(cb) as usize; - let code_size = end_ptr - start_ptr; - let iseq_name = iseq_get_location(iseq, 0); - register_with_perf(format!("entry for {iseq_name}"), start_ptr, code_size); + // Compile the ISEQ. When function is None, this is a lazy compile + // from a stub hit -- wrap in a trace event covering the full compile. + let mut version = IseqVersion::new(iseq); + let code_ptrs = if function.is_none() { + trace_compile_phase(&iseq_get_location(iseq, 0), || gen_iseq_body(cb, iseq, version, function)) + } else { + gen_iseq_body(cb, iseq, version, function) + }; + match &code_ptrs { + Ok(code_ptrs) => { + unsafe { version.as_mut() }.status = IseqStatus::Compiled(code_ptrs.clone()); + incr_counter!(compiled_iseq_count); + } + Err(err) => { + unsafe { version.as_mut() }.status = IseqStatus::CantCompile(err.clone()); + incr_counter!(failed_iseq_count); } } - result + payload.versions.push(version); + code_ptrs } /// Compile an ISEQ into machine code -fn gen_iseq(cb: &mut CodeBlock, iseq: IseqPtr) -> Option<(CodePtr, Vec<Rc<RefCell<IseqCall>>>)> { - // Return an existing pointer if it's already compiled - let payload = get_or_create_iseq_payload(iseq); - match payload.status { - IseqStatus::Compiled(start_ptr) => return Some((start_ptr, vec![])), - IseqStatus::CantCompile => return None, - IseqStatus::NotCompiled => {}, +fn gen_iseq_body(cb: &mut CodeBlock, iseq: IseqPtr, mut version: IseqVersionRef, function: Option<&Function>) -> Result<IseqCodePtrs, CompileError> { + // If we ran out of code region, we shouldn't attempt to generate new code. + if cb.has_dropped_bytes() { + return Err(CompileError::OutOfMemory); } - // Convert ISEQ into High-level IR and optimize HIR - let function = match compile_iseq(iseq) { + // Convert ISEQ into optimized High-level IR if not given + let function = match function { Some(function) => function, - None => { - payload.status = IseqStatus::CantCompile; - return None; - } + None => &crate::stats::with_time_stat(Counter::compile_hir_time_ns, || compile_iseq(iseq))?, }; // Compile the High-level IR - let result = gen_function(cb, iseq, &function); - if let Some((start_ptr, gc_offsets, jit)) = result { - payload.status = IseqStatus::Compiled(start_ptr); - payload.iseq_calls.extend(jit.iseq_calls.clone()); - append_gc_offsets(iseq, &gc_offsets); - Some((start_ptr, jit.iseq_calls)) - } else { - payload.status = IseqStatus::CantCompile; - None - } -} + let (iseq_code_ptrs, gc_offsets, iseq_calls) = + trace_compile_phase("codegen", || { + let (iseq_code_ptrs, gc_offsets, iseq_calls) = + crate::stats::with_time_stat(Counter::compile_lir_time_ns, || gen_function(cb, iseq, version, function))?; + + // Stub callee ISEQs for JIT-to-JIT calls + trace_compile_phase("generate_jit_jit_stubs", || { + for iseq_call in iseq_calls.iter() { + gen_iseq_call(cb, iseq_call)?; + } + Ok::<(), CompileError>(()) + })?; -/// Compile a function -fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function) -> Option<(CodePtr, Vec<CodePtr>, JITState)> { - let c_stack_slots = max_num_params(function).saturating_sub(ALLOC_REGS.len()); - let mut jit = JITState::new(iseq, function.num_insns(), function.num_blocks(), c_stack_slots); - let mut asm = Assembler::new(); + Ok((iseq_code_ptrs, gc_offsets, iseq_calls)) + })?; - // Compile each basic block - let reverse_post_order = function.rpo(); - for &block_id in reverse_post_order.iter() { - let block = function.block(block_id); - asm_comment!( - asm, "{block_id}({}): {}", - block.params().map(|param| format!("{param}")).collect::<Vec<_>>().join(", "), - iseq_get_location(iseq, block.insn_idx), - ); - - // Write a label to jump to the basic block - let label = jit.get_label(&mut asm, block_id); - asm.write_label(label); + // Prepare for GC + unsafe { version.as_mut() }.outgoing.extend(iseq_calls); + append_gc_offsets(iseq, version, &gc_offsets); + Ok(iseq_code_ptrs) +} - // Set up the frame at the first block. :bb0-prologue: - if block_id == BlockId(0) { - asm.frame_setup(&[], jit.c_stack_slots); +/// Compile a function +fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, function: &Function) -> Result<(IseqCodePtrs, Vec<CodePtr>, Vec<IseqCallRef>), CompileError> { + let (mut jit, asm) = trace_compile_phase("codegen", || { + let mut jit = JITState::new(version, function.num_insns(), function.num_blocks(), JIT_FRAME_SIZE); + let mut asm = Assembler::new_with_stack_slots(JIT_FRAME_SIZE); + + // Mapping from HIR block IDs to LIR block IDs. + // This is is a one-to-one mapping from HIR to LIR blocks used for finding + // jump targets in LIR (LIR should always jump to the head of an HIR block) + let mut hir_to_lir: Vec<Option<lir::BlockId>> = vec![None; function.num_blocks()]; + + let reverse_post_order = function.reverse_post_order(); + + // Create all LIR basic blocks corresponding to HIR basic blocks + for (rpo_idx, &block_id) in reverse_post_order.iter().enumerate() { + // Skip the entries superblock -- it's an internal CFG artifact + if block_id == function.entries_block { continue; } + let lir_block_id = asm.new_block(block_id, function.is_entry_block(block_id), rpo_idx); + hir_to_lir[block_id.0] = Some(lir_block_id); } - // Compile all parameters - for &insn_id in block.params() { - match function.find(insn_id) { - Insn::Param { idx } => { - jit.opnds[insn_id.0] = Some(gen_param(&mut asm, idx)); - }, - insn => unreachable!("Non-param insn found in block.params: {insn:?}"), + // Compile each basic block + for &block_id in reverse_post_order.iter() { + // Skip the entries superblock -- it's an internal CFG artifact + if block_id == function.entries_block { continue; } + // Set the current block to the LIR block that corresponds to this + // HIR block. + let lir_block_id = hir_to_lir[block_id.0].unwrap(); + asm.set_current_block(lir_block_id); + + // Write a label to jump to the basic block + let label = jit.get_label(&mut asm, lir_block_id, block_id); + asm.write_label(label); + + let block = function.block(block_id); + asm_comment!( + asm, "{block_id}({}): {}", + block.params().map(|param| format!("{param}")).collect::<Vec<_>>().join(", "), + iseq_get_location(iseq, block.insn_idx), + ); + + // Compile all parameters + for (idx, &insn_id) in block.params().enumerate() { + match function.find(insn_id) { + Insn::Param => { + jit.opnds[insn_id.0] = Some(gen_param(&mut asm, idx)); + }, + insn => unreachable!("Non-param insn found in block.params: {insn:?}"), + } + } + + // In JIT entry blocks, compile LoadArg instructions before other instructions + // so that calling convention registers are reserved early, like Param. + if function.is_entry_block(block_id) { + for &insn_id in block.insns() { + if let Insn::LoadArg { idx, .. } = function.find(insn_id) { + jit.opnds[insn_id.0] = Some(gen_param(&mut asm, idx as usize)); + } + } } - } - // Compile all instructions - for &insn_id in block.insns() { - let insn = function.find(insn_id); - if gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn).is_none() { - debug!("Failed to compile insn: {insn_id} {insn}"); - return None; + // Compile all instructions + for (insn_idx, &insn_id) in block.insns().enumerate() { + let insn = function.find(insn_id); + + match insn { + Insn::CondBranch { val, if_true, if_false } => { + let val_opnd = jit.get_opnd(val); + let true_target = hir_to_lir[if_true.target.0].unwrap(); + let false_target = hir_to_lir[if_false.target.0].unwrap(); + + let true_branch = lir::BranchEdge { + target: true_target, + args: if_true.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + let false_branch = lir::BranchEdge { + target: false_target, + args: if_false.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + + asm.test(val_opnd, val_opnd); + asm.push_insn(lir::Insn::Jnz(Target::Block(true_branch))); + asm.jmp(Target::Block(false_branch)); + + assert!(asm.current_block().insns.last().unwrap().is_terminator()); + } + Insn::Jump(target) => { + let lir_target = hir_to_lir[target.target.0].unwrap(); + let branch_edge = lir::BranchEdge { + target: lir_target, + args: target.args.iter().map(|insn_id| jit.get_opnd(*insn_id)).collect() + }; + asm.jmp(Target::Block(branch_edge)); + assert!(asm.current_block().insns.last().unwrap().is_terminator()); + + // Jump should always be the last instruction in an HIR block + assert!(insn_idx == block.insns().len() - 1, "Jump must be the last instruction in HIR block"); + }, + _ => { + // Start a new perf range for the HIR instruction. For now, we do this only for + // non-terminator instructions because LIR blocks must end with a terminator instruction. + let perf_symbol = if get_option!(perf) == Some(PerfMap::HIR) && !insn.is_terminator() { + let insn_name = format!("{insn}").split_whitespace().next().unwrap().to_string(); + Some(perf_symbol_range_start(&mut asm, &insn_name)) + } else { + None + }; + + let result = gen_insn(cb, &mut jit, &mut asm, function, insn_id, &insn); + + // Close the current perf range for the HIR instruction. + if let Some(perf_symbol) = &perf_symbol { + perf_symbol_range_end(&mut asm, perf_symbol); + } + + if let Err(last_snapshot) = result { + debug!("ZJIT: gen_function: Failed to compile insn: {insn_id} {insn}. Generating side-exit."); + gen_incr_counter(&mut asm, exit_counter_for_unhandled_hir_insn(&insn)); + let reason = match insn { + Insn::Throw { .. } => SideExitReason::UnhandledHIRThrow, + Insn::InvokeBuiltin { .. } => SideExitReason::UnhandledHIRInvokeBuiltin, + _ => SideExitReason::UnhandledHIRUnknown(insn_id), + }; + gen_side_exit(&mut jit, &mut asm, &reason, None, &function.frame_state(last_snapshot)); + // Don't bother generating code after a side-exit. We won't run it. + // TODO(max): Generate ud2 or equivalent. + break; + }; + // It's fine; we generated the instruction + } + } } + // Blocks should always end with control flow + assert!(asm.current_block().insns.last().unwrap().is_terminator()); } - // Make sure the last patch point has enough space to insert a jump - asm.pad_patch_point(); - } - if get_option!(dump_lir) { - println!("LIR:\nfn {}:\n{:?}", iseq_name(iseq), asm); - } + assert!(!asm.reverse_post_order().is_empty()); + + // Validate CFG invariants after HIR to LIR lowering + asm.validate_jump_positions(); + + (jit, asm) + }); // Generate code if everything can be compiled - let result = asm.compile(cb).map(|(start_ptr, gc_offsets)| (start_ptr, gc_offsets, jit)); - if let Some((start_ptr, _, _)) = result { - if get_option!(perf) { - let start_usize = start_ptr.raw_ptr(cb) as usize; - let end_usize = cb.get_write_ptr().raw_ptr(cb) as usize; + let result = asm.compile(cb); + if let Ok((start_ptr, _)) = result { + if get_option!(perf) == Some(PerfMap::ISEQ) { + let start_usize = start_ptr.raw_addr(cb); + let end_usize = cb.get_write_ptr().raw_addr(cb); let code_size = end_usize - start_usize; let iseq_name = iseq_get_location(iseq, 0); register_with_perf(iseq_name, start_usize, code_size); @@ -305,11 +553,19 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, function: &Function) -> Optio ZJITState::log_compile(iseq_name); } } - result + result.map(|(start_ptr, gc_offsets)| { + // Make sure jit_entry_ptrs can be used as a parallel vector to jit_entry_insns() + jit.jit_entries.sort_by_key(|jit_entry| jit_entry.borrow().jit_entry_idx); + + let jit_entry_ptrs = jit.jit_entries.iter().map(|jit_entry| + jit_entry.borrow().start_addr.get().expect("start_addr should have been set by pos_marker in gen_entry_point") + ).collect(); + (IseqCodePtrs { start_ptr, jit_entry_ptrs }, gc_offsets, jit.iseq_calls) + }) } /// Compile an instruction -fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, function: &Function, insn_id: InsnId, insn: &Insn) -> Option<()> { +fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, function: &Function, insn_id: InsnId, insn: &Insn) -> Result<(), InsnId> { // Convert InsnId to lir::Opnd macro_rules! opnd { ($insn_id:ident) => { @@ -327,7 +583,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio macro_rules! no_output { ($call:expr) => { - { let () = $call; return Some(()); } + { let () = $call; return Ok(()); } }; } @@ -336,36 +592,76 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } let out_opnd = match insn { - Insn::Const { val: Const::Value(val) } => gen_const(*val), - Insn::NewArray { elements, state } => gen_new_array(asm, opnds!(elements), &function.frame_state(*state)), - Insn::NewHash { elements, state } => gen_new_hash(jit, asm, elements, &function.frame_state(*state)), + Insn::Comment { .. } => return Ok(()), // comment instruction, no code generation + &Insn::Const { val: Const::Value(val) } => gen_const_value(val), + &Insn::Const { val: Const::CPtr(val) } => gen_const_cptr(val), + &Insn::Const { val: Const::CInt64(val) } => gen_const_long(val), + &Insn::Const { val: Const::CUInt16(val) } => gen_const_uint16(val), + &Insn::Const { val: Const::CUInt32(val) } => gen_const_uint32(val), + &Insn::Const { val: Const::CUInt64(val) } => Opnd::UImm(val), + &Insn::Const { val: Const::CAttrIndex(val) } => gen_const_attr_index_t(val), + &Insn::Const { val: Const::CShape(val) } => { + assert_eq!(SHAPE_ID_NUM_BITS, 32); + gen_const_uint32(val.0) + } + Insn::Const { .. } => panic!("Unexpected Const in gen_insn: {insn}"), + Insn::NewArray { elements, state } => gen_new_array(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::NewHash { elements, state } => gen_new_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::ArrayDup { val, state } => gen_array_dup(asm, opnd!(val), &function.frame_state(*state)), + Insn::AdjustBounds { index, length } => gen_adjust_bounds(asm, opnd!(index), opnd!(length)), + Insn::ArrayAref { array, index, .. } => gen_array_aref(asm, opnd!(array), opnd!(index)), + Insn::ArrayAset { array, index, val } => { + no_output!(gen_array_aset(asm, opnd!(array), opnd!(index), opnd!(val))) + } + Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), + Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), + Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, opnd!(val), &function.frame_state(*state)), + &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(asm, class, &function.frame_state(state)), Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), // concatstrings shouldn't have 0 strings // If it happens we abort the compilation for now - Insn::StringConcat { strings, .. } if strings.is_empty() => return None, + Insn::StringConcat { strings, state, .. } if strings.is_empty() => return Err(*state), Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)), + &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), + Insn::StringSetbyteFixnum { string, index, value } => gen_string_setbyte_fixnum(asm, opnd!(string), opnd!(index), opnd!(value)), + Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringEqual { left, right } => gen_string_equal(asm, opnd!(left), opnd!(right)), Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)), Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), - Insn::Param { idx } => unreachable!("block.insns should not have Insn::Param({idx})"), - Insn::Snapshot { .. } => return Some(()), // we don't need to do anything for this instruction at the moment - Insn::Jump(branch) => no_output!(gen_jump(jit, asm, branch)), - Insn::IfTrue { val, target } => no_output!(gen_if_true(jit, asm, opnd!(val), target)), - Insn::IfFalse { val, target } => no_output!(gen_if_false(jit, asm, opnd!(val), target)), - Insn::SendWithoutBlock { cd, state, .. } => gen_send_without_block(jit, asm, *cd, &function.frame_state(*state)), - // Give up SendWithoutBlockDirect for 6+ args since asm.ccall() doesn't support it. - Insn::SendWithoutBlockDirect { cd, state, args, .. } if args.len() + 1 > C_ARG_OPNDS.len() => // +1 for self - gen_send_without_block(jit, asm, *cd, &function.frame_state(*state)), - Insn::SendWithoutBlockDirect { cme, iseq, self_val, args, state, .. } => gen_send_without_block_direct(cb, jit, asm, *cme, *iseq, opnd!(self_val), opnds!(args), &function.frame_state(*state)), + Insn::Param => unreachable!("block.insns should not have Insn::Param"), + Insn::LoadArg { .. } => return Ok(()), // compiled in the LoadArg pre-pass above + Insn::Snapshot { .. } => return Ok(()), // we don't need to do anything for this instruction at the moment + &Insn::Send { cd, block: None, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockIseq(blockiseq)), state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockArg), state, reason, .. } => gen_send(jit, asm, cd, std::ptr::null(), &function.frame_state(state), reason), + &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::SendDirect { cme, iseq, recv, ref args, kw_bits, block, state, .. } => gen_send_iseq_direct( + cb, jit, asm, cme, iseq, opnd!(recv), opnds!(args), + kw_bits, &function.frame_state(state), block, + ), + &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), + Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), + Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), // Ensure we have enough room fit ec, self, and arguments // TODO remove this check when we have stack args (we can use Time.new to test it) - Insn::InvokeBuiltin { bf, .. } if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) => return None, - Insn::InvokeBuiltin { bf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, opnds!(args)), + Insn::InvokeBuiltin { bf, state, .. } if bf.argc + 2 > (C_ARG_OPNDS.len() as i32) => return Err(*state), + Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, *leaf, opnds!(args)), + &Insn::EntryPoint { jit_entry_idx } => no_output!(gen_entry_point(jit, asm, jit_entry_idx)), Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FloatAdd { recv, other, state } => gen_float_add(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatSub { recv, other, state } => gen_float_sub(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatMul { recv, other, state } => gen_float_mul(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatDiv { recv, other, state } => gen_float_div(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::FloatToInt { recv, state } => gen_float_to_int(asm, opnd!(recv), &function.frame_state(*state)), Insn::FixnumEq { left, right } => gen_fixnum_eq(asm, opnd!(left), opnd!(right)), Insn::FixnumNeq { left, right } => gen_fixnum_neq(asm, opnd!(left), opnd!(right)), Insn::FixnumLt { left, right } => gen_fixnum_lt(asm, opnd!(left), opnd!(right)), @@ -374,44 +670,109 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::FixnumGe { left, right } => gen_fixnum_ge(asm, opnd!(left), opnd!(right)), Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right)), Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right)), - Insn::IsNil { val } => gen_isnil(asm, opnd!(val)), + Insn::FixnumXor { left, right } => gen_fixnum_xor(asm, opnd!(left), opnd!(right)), + Insn::IntAnd { left, right } => asm.and(opnd!(left), opnd!(right)), + Insn::IntOr { left, right } => gen_int_or(asm, opnd!(left), opnd!(right)), + &Insn::FixnumLShift { left, right, state } => { + // We only create FixnumLShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_lshift(jit, asm, opnd!(left), shift_amount, &function.frame_state(state)) + } + &Insn::FixnumRShift { left, right } => { + // We only create FixnumRShift when we know the shift amount statically and it's in [0, + // 63]. + let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; + gen_fixnum_rshift(asm, opnd!(left), shift_amount) + } + &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::FixnumAref { recv, index } => gen_fixnum_aref(asm, opnd!(recv), opnd!(index)), + &Insn::IsMethodCfunc { val, cd, cfunc, state } => gen_is_method_cfunc(asm, opnd!(val), cd, cfunc, &function.frame_state(state)), + &Insn::IsBitEqual { left, right } => gen_is_bit_equal(asm, opnd!(left), opnd!(right)), + &Insn::IsBitNotEqual { left, right } => gen_is_bit_not_equal(asm, opnd!(left), opnd!(right)), + &Insn::BoxBool { val } => gen_box_bool(asm, opnd!(val)), + &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, opnd!(val), &function.frame_state(state)), + &Insn::UnboxFixnum { val } => gen_unbox_fixnum(asm, opnd!(val)), Insn::Test { val } => gen_test(asm, opnd!(val)), - Insn::GuardType { val, guard_type, state } => gen_guard_type(jit, asm, opnd!(val), *guard_type, &function.frame_state(*state)), - Insn::GuardBitEquals { val, expected, state } => gen_guard_bit_equals(jit, asm, opnd!(val), *expected, &function.frame_state(*state)), + Insn::RefineType { val, .. } => opnd!(val), + Insn::HasType { val, expected } => { + let val_type = function.type_of(*val); + gen_has_type(jit, asm, opnd!(val), val_type, *expected) + } + &Insn::GuardType { val, guard_type, state, recompile } => { + let val_type = function.type_of(val); + gen_guard_type(jit, asm, opnd!(val), val_type, guard_type, recompile, &function.frame_state(state)) + } + &Insn::GuardBitEquals { val, expected, reason, state, recompile } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, reason, recompile, &function.frame_state(state)), + &Insn::GuardAnyBitSet { val, mask, reason, state, .. } => gen_guard_any_bit_set(jit, asm, opnd!(val), mask, reason, &function.frame_state(state)), + &Insn::GuardNoBitsSet { val, mask, reason, state, .. } => gen_guard_no_bits_set(jit, asm, opnd!(val), mask, reason, &function.frame_state(state)), + &Insn::GuardLess { left, right, reason, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), reason, &function.frame_state(state)), + &Insn::GuardGreaterEq { left, right, state, .. } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), - Insn::CCall { cfun, args, name: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfun, opnds!(args)), - Insn::GetIvar { self_val, id, state: _ } => gen_getivar(asm, opnd!(self_val), *id), + Insn::CCall { cfunc, recv, args, name, owner: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), + // Give up CCallWithFrame for 7+ args since asm.ccall() supports at most 6 args (recv + args). + // We're currently emitting a CCallWithFrame for `super` in to a cfunction. + // We can't lower to `gen_send_without_block` because the + // source opcode isn't necessarily `opt_send_without_block` + // and so the interpreter stack layout may be incompatible. + Insn::CCallWithFrame { cd, state, args, block, .. } if args.len() + 1 > C_ARG_OPNDS.len() => return Err(*state), + Insn::CCallWithFrame { cfunc, recv, name, args, cme, state, block, .. } => + gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)), + Insn::CCallVariadic { cfunc, recv, name, args, cme, state, block, return_type: _, elidable: _ } => { + gen_ccall_variadic(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) + } + Insn::GetIvar { self_val, id, ic, state } => gen_getivar(asm, opnd!(self_val), *id, *ic, &function.frame_state(*state)), Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), - Insn::GetGlobal { id, state: _ } => gen_getglobal(asm, *id), - &Insn::GetLocal { ep_offset, level } => gen_getlocal_with_ep(asm, ep_offset, level), - &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal_with_ep(asm, opnd!(val), function.type_of(val), ep_offset, level)), + Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), + &Insn::IsBlockParamModified { flags } => gen_is_block_param_modified(asm, opnd!(flags)), + &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, ep_offset, level, &function.frame_state(state)), + &Insn::SetLocal { val, ep_offset, level } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), + Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), - Insn::SetIvar { self_val, id, val, state: _ } => no_output!(gen_setivar(asm, opnd!(self_val), *id, opnd!(val))), - Insn::SideExit { state, reason } => no_output!(gen_side_exit(jit, asm, reason, &function.frame_state(*state))), - Insn::PutSpecialObject { value_type } => gen_putspecialobject(asm, *value_type), + Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, *id, *ic, &function.frame_state(*state)), + Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, *id, opnd!(val), *ic, &function.frame_state(*state))), + Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), + Insn::FixnumBitCheck { val, index } => gen_fixnum_bit_check(asm, opnd!(val), *index), + Insn::SideExit { state, reason, recompile } => no_output!(gen_side_exit(jit, asm, reason, *recompile, &function.frame_state(*state))), + Insn::PutSpecialObject { value_type, state } => gen_putspecialobject(jit, asm, *value_type, &function.frame_state(*state)), Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)), - Insn::Defined { op_type, obj, pushval, v, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), &function.frame_state(*state)), - Insn::GetSpecialSymbol { symbol_type, state: _ } => gen_getspecial_symbol(asm, *symbol_type), + Insn::Defined { op_type, obj, pushval, v, lep_level, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), *lep_level, &function.frame_state(*state)), + Insn::CheckMatch { target, pattern, flag, state } => gen_checkmatch(jit, asm, opnd!(target), opnd!(pattern), *flag, &function.frame_state(*state)), + Insn::GetSpecialSymbol { symbol_type, state } => gen_getspecial_symbol(asm, *symbol_type, &function.frame_state(*state)), Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)), &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)), + Insn::IncrCounterPtr { counter_ptr } => no_output!(gen_incr_counter_ptr(asm, *counter_ptr)), Insn::ObjToString { val, cd, state, .. } => gen_objtostring(jit, asm, opnd!(val), *cd, &function.frame_state(*state)), &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, &function.frame_state(state))), - Insn::ArrayExtend { .. } - | Insn::ArrayMax { .. } - | Insn::ArrayPush { .. } - | Insn::DefinedIvar { .. } - | Insn::FixnumDiv { .. } - | Insn::FixnumMod { .. } - | Insn::HashDup { .. } - | Insn::Send { .. } - | Insn::Throw { .. } - | Insn::ToArray { .. } - | Insn::ToNewArray { .. } - | Insn::Const { .. } - => { - debug!("ZJIT: gen_function: unexpected insn {insn}"); - return None; - } + Insn::BreakPoint => no_output!(asm.breakpoint()), + Insn::Unreachable => no_output!(asm.abort()), + &Insn::HashDup { val, state } => { gen_hash_dup(asm, opnd!(val), &function.frame_state(state)) }, + &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, opnd!(hash), opnd!(key), &function.frame_state(state)) }, + &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, + &Insn::ArrayPush { array, val, state } => { no_output!(gen_array_push(asm, opnd!(array), opnd!(val), &function.frame_state(state))) }, + &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::ToArray { val, state } => { gen_to_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::DefinedIvar { self_val, id, pushval, .. } => { gen_defined_ivar(asm, opnd!(self_val), id, pushval) }, + &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state))) }, + Insn::LoadPC => gen_load_pc(asm), + Insn::LoadEC => gen_load_ec(), + Insn::LoadSP => gen_load_sp(), + &Insn::GetEP { level } => gen_get_ep(asm, level), + Insn::LoadSelf => gen_load_self(), + &Insn::LoadField { recv, id, offset, return_type } => gen_load_field(asm, opnd!(recv), id, offset, return_type), + &Insn::StoreField { recv, id, offset, val } => no_output!(gen_store_field(asm, opnd!(recv), id, offset, opnd!(val), function.type_of(val))), + &Insn::WriteBarrier { recv, val } => no_output!(gen_write_barrier(jit, asm, opnd!(recv), opnd!(val), function.type_of(val))), + &Insn::IsBlockGiven { lep } => gen_is_block_given(asm, opnd!(lep)), + Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, opnds!(elements), opnd!(target), &function.frame_state(*state)), + Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, opnds!(elements), opnd!(fmt), (*buffer).map(|buffer| opnd!(buffer)), &function.frame_state(*state)), + &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, ary, opnd!(target), &function.frame_state(state)), + Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + &Insn::IsA { val, class } => gen_is_a(jit, asm, opnd!(val), opnd!(class)), + &Insn::ArrayMax { ref elements, state } => gen_array_max(jit, asm, opnds!(elements), &function.frame_state(state)), + &Insn::ArrayMin { ref elements, state } => gen_array_min(jit, asm, opnds!(elements), &function.frame_state(state)), + &Insn::Throw { state, .. } => return Err(state), + &Insn::CondBranch { .. } + | &Insn::Jump { .. } | Insn::Entries { .. } => unreachable!(), }; assert!(insn.has_output(), "Cannot write LIR output of HIR instruction with no output: {insn}"); @@ -419,26 +780,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio // If the instruction has an output, remember it in jit.opnds jit.opnds[insn_id.0] = Some(out_opnd); - Some(()) -} - -/// Gets the EP of the ISeq of the containing method, or "local level". -/// Equivalent of GET_LEP() macro. -fn gen_get_lep(jit: &JITState, asm: &mut Assembler) -> Opnd { - // Equivalent of get_lvar_level() in compile.c - fn get_lvar_level(mut iseq: IseqPtr) -> u32 { - let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; - let mut level = 0; - while iseq != local_iseq { - iseq = unsafe { rb_get_iseq_body_parent_iseq(iseq) }; - level += 1; - } - - level - } - - let level = get_lvar_level(jit.iseq); - gen_get_ep(asm, level) + Ok(()) } // Get EP at `level` from CFP @@ -462,38 +804,29 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *const rb_call_data, state: &FrameState) -> Opnd { gen_prepare_non_leaf_call(jit, asm, state); - - let iseq_opnd = Opnd::Value(jit.iseq.into()); - // TODO: Specialize for immediate types - // Call rb_vm_objtostring(iseq, recv, cd) - let ret = asm_ccall!(asm, rb_vm_objtostring, iseq_opnd, val, (cd as usize).into()); + // Call rb_vm_objtostring(cfp, recv, cd) + let ret = asm_ccall!(asm, rb_vm_objtostring, CFP, val, Opnd::const_ptr(cd)); // TODO: Call `to_s` on the receiver if rb_vm_objtostring returns Qundef // Need to replicate what CALL_SIMPLE_METHOD does asm_comment!(asm, "side-exit if rb_vm_objtostring returns Qundef"); asm.cmp(ret, Qundef.into()); - asm.je(side_exit(jit, state, ObjToStringFallback)); + asm.je(jit, side_exit(jit, state, ObjToStringFallback)); ret } -fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, state: &FrameState) -> Opnd { +fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, lep_level: u32, state: &FrameState) -> Opnd { match op_type as defined_type { DEFINED_YIELD => { - // `yield` goes to the block handler stowed in the "local" iseq which is - // the current iseq or a parent. Only the "method" iseq type can be passed a - // block handler. (e.g. `yield` in the top level script is a syntax error.) - let local_iseq = unsafe { rb_get_iseq_body_local_iseq(jit.iseq) }; - if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { - let lep = gen_get_lep(jit, asm); - let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); - let pushval = asm.load(pushval.into()); - asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); - asm.csel_e(Qnil.into(), pushval.into()) - } else { - Qnil.into() - } + // `lep_level` was precomputed at HIR construction so we can materialize the local EP + // inline without walking the parent iseq chain here. + let lep = gen_get_ep(asm, lep_level); + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + let pushval = asm.load(pushval.into()); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qnil.into(), pushval) } _ => { // Save the PC and SP because the callee may allocate or call #respond_to? @@ -509,34 +842,90 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, } } -/// Get a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. -/// We generate this instruction with level=0 only when the local variable is on the heap, so we -/// can't optimize the level=0 case using the SP register. -fn gen_getlocal_with_ep(asm: &mut Assembler, local_ep_offset: u32, level: u32) -> lir::Opnd { - let ep = gen_get_ep(asm, level); - let offset = -(SIZEOF_VALUE_I32 * i32::try_from(local_ep_offset).expect(&format!("Could not convert local_ep_offset {local_ep_offset} to i32"))); - asm.load(Opnd::mem(64, ep, offset)) +/// Similar to gen_defined for DEFINED_YIELD +fn gen_is_block_given(asm: &mut Assembler, lep: Opnd) -> Opnd { + let block_handler = asm.load(Opnd::mem(64, lep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + asm.cmp(block_handler, VM_BLOCK_HANDLER_NONE.into()); + asm.csel_e(Qfalse.into(), Qtrue.into()) +} + +fn gen_unbox_fixnum(asm: &mut Assembler, val: Opnd) -> Opnd { + asm.rshift(val, Opnd::UImm(1)) } /// Set a local variable from a higher scope or the heap. `local_ep_offset` is in number of VALUEs. /// We generate this instruction with level=0 only when the local variable is on the heap, so we /// can't optimize the level=0 case using the SP register. -fn gen_setlocal_with_ep(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: u32, level: u32) { +fn gen_setlocal(asm: &mut Assembler, val: Opnd, val_type: Type, local_ep_offset: u32, level: u32) { + let local_ep_offset = c_int::try_from(local_ep_offset).unwrap_or_else(|_| panic!("Could not convert local_ep_offset {local_ep_offset} to i32")); + if level > 0 { + gen_incr_counter(asm, Counter::vm_write_to_parent_iseq_local_count); + } let ep = gen_get_ep(asm, level); // When we've proved that we're writing an immediate, // we can skip the write barrier. if val_type.is_immediate() { - let offset = -(SIZEOF_VALUE_I32 * i32::try_from(local_ep_offset).expect(&format!("Could not convert local_ep_offset {local_ep_offset} to i32"))); + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); asm.mov(Opnd::mem(64, ep, offset), val); } else { // We're potentially writing a reference to an IMEMO/env object, // so take care of the write barrier with a function. - let local_index = c_int::try_from(local_ep_offset).ok().and_then(|idx| idx.checked_mul(-1)).expect(&format!("Could not turn {local_ep_offset} into a negative c_int")); + let local_index = -local_ep_offset; asm_ccall!(asm, rb_vm_env_write, ep, local_index.into(), val); } } +/// Returns 1 (as CBool) when VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set; returns 0 otherwise. +fn gen_is_block_param_modified(asm: &mut Assembler, flags: Opnd) -> Opnd { + asm.test(flags, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.csel_nz(Opnd::Imm(1), Opnd::Imm(0)) +} + +/// Get the block parameter as a Proc, write it to the environment, +/// and mark the flag as modified. +fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + // Bail out if write barrier is required. + let ep = gen_get_ep(asm, level); + let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + asm.test(flags, VM_ENV_FLAG_WB_REQUIRED.into()); + asm.jnz(jit, side_exit(jit, state, SideExitReason::BlockParamWbRequired)); + + // Convert block handler to Proc. + let block_handler = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); + let proc = asm_ccall!(asm, rb_vm_bh_to_procval, EC, block_handler); + + // Write Proc to EP and mark modified. + let ep = gen_get_ep(asm, level); + let local_ep_offset = c_int::try_from(ep_offset).unwrap_or_else(|_| { + panic!("Could not convert local_ep_offset {ep_offset} to i32") + }); + let offset = -(SIZEOF_VALUE_I32 * local_ep_offset); + asm.mov(Opnd::mem(VALUE_BITS, ep, offset), proc); + + let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); + let flags_val = asm.load(flags); + let modified = asm.or(flags_val, VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()); + asm.store(flags, modified); + + // Read the Proc from EP. + let ep = gen_get_ep(asm, level); + asm.load(Opnd::mem(VALUE_BITS, ep, offset)) +} + +fn gen_guard_less(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, reason: SideExitReason, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jge(jit, side_exit(jit, state, reason)); + left +} + +fn gen_guard_greater_eq(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { + asm.cmp(left, right); + asm.jl(jit, side_exit(jit, state, SideExitReason::GuardGreaterEq)); + left +} + fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { unsafe extern "C" { fn rb_vm_opt_getconstant_path(ec: EcPtr, cfp: CfpPtr, ic: *const iseq_inline_constant_cache) -> VALUE; @@ -548,76 +937,296 @@ fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_in asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic)) } -fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, args: Vec<Opnd>) -> lir::Opnd { +fn gen_getconstant(jit: &mut JITState, asm: &mut Assembler, klass: Opnd, id: ID, allow_nil: Opnd, state: &FrameState) -> Opnd { + unsafe extern "C" { + fn rb_vm_get_ev_const(ec: EcPtr, klass: VALUE, id: ID, allow_nil: VALUE) -> VALUE; + } + + // Constant lookup can raise and run arbitrary Ruby code via const_missing. + gen_prepare_non_leaf_call(jit, asm, state); + + asm_ccall!(asm, rb_vm_get_ev_const, EC, klass, id.0.into(), allow_nil) +} + +fn gen_fixnum_bit_check(asm: &mut Assembler, val: Opnd, index: u8) -> Opnd { + let bit_test: u64 = 0x01 << (index + 1); + asm.test(val, bit_test.into()); + asm.csel_z(Qtrue.into(), Qfalse.into()) +} + +fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec<Opnd>) -> lir::Opnd { assert!(bf.argc + 2 <= C_ARG_OPNDS.len() as i32, "gen_invokebuiltin should not be called for builtin function {} with too many arguments: {}", unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }, bf.argc); - // Anything can happen inside builtin functions - gen_prepare_non_leaf_call(jit, asm, state); + if leaf { + gen_prepare_leaf_call_with_gc(asm, state); + } else { + // Anything can happen inside builtin functions + gen_prepare_non_leaf_call(jit, asm, state); + } let mut cargs = vec![EC]; cargs.extend(args); + asm.count_call_to(unsafe { std::ffi::CStr::from_ptr(bf.name).to_str().unwrap() }); asm.ccall(bf.func_ptr as *const u8, cargs) } /// Record a patch point that should be invalidated on a given invariant fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { - let payload_ptr = get_or_create_iseq_payload_ptr(jit.iseq); - let label = asm.new_label("patch_point").unwrap_label(); - let invariant = invariant.clone(); + let invariant = *invariant; + let exit = build_side_exit(jit, state); + + // Let compile_exits compile a side exit. Let scratch_split lower it with split_patch_point. + asm.patch_point(Target::SideExit { exit, reason: PatchPoint(invariant) }, invariant, jit.version); +} + +/// This is used by scratch_split to lower PatchPoint into PadPatchPoint and PosMarker. +/// It's called at scratch_split so that we can use the Label after side-exit deduplication in compile_exits. +pub fn split_patch_point(asm: &mut Assembler, target: &Target, invariant: Invariant, version: IseqVersionRef) { + let Target::Label(exit_label) = *target else { + unreachable!("PatchPoint's target should have been lowered to Target::Label by compile_exits: {target:?}"); + }; - // Compile a side exit. Fill nop instructions if the last patch point is too close. - asm.patch_point(build_side_exit(jit, state, PatchPoint(invariant), Some(label))); + // Fill nop instructions if the last patch point is too close. + asm.pad_patch_point(); // Remember the current address as a patch point asm.pos_marker(move |code_ptr, cb| { + let side_exit_ptr = cb.resolve_label(exit_label); match invariant { Invariant::BOPRedefined { klass, bop } => { - let side_exit_ptr = cb.resolve_label(label); - track_bop_assumption(klass, bop, code_ptr, side_exit_ptr, payload_ptr); + track_bop_assumption(klass, bop, code_ptr, side_exit_ptr, version); } Invariant::MethodRedefined { klass: _, method: _, cme } => { - let side_exit_ptr = cb.resolve_label(label); - track_cme_assumption(cme, code_ptr, side_exit_ptr, payload_ptr); + track_cme_assumption(cme, code_ptr, side_exit_ptr, version); } Invariant::StableConstantNames { idlist } => { - let side_exit_ptr = cb.resolve_label(label); - track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr, payload_ptr); + track_stable_constant_names_assumption(idlist, code_ptr, side_exit_ptr, version); + } + Invariant::NoTracePoint => { + track_no_trace_point_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoEPEscape(iseq) => { + track_no_ep_escape_assumption(iseq, code_ptr, side_exit_ptr, version); } Invariant::SingleRactorMode => { - let side_exit_ptr = cb.resolve_label(label); - track_single_ractor_assumption(code_ptr, side_exit_ptr, payload_ptr); + track_single_ractor_assumption(code_ptr, side_exit_ptr, version); + } + Invariant::NoSingletonClass { klass } => { + track_no_singleton_class_assumption(klass, code_ptr, side_exit_ptr, version); + } + Invariant::RootBoxOnly => { + track_root_box_assumption(code_ptr, side_exit_ptr, version); } } }); } +/// Generate code for a C function call that pushes a frame +fn gen_ccall_with_frame( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + block: Option<BlockHandler>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::non_variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + let caller_stack_size = state.stack().len() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (and block arguments if any) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(BlockHandler::BlockIseq(block_iseq)) = block { + // Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). + // VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. + // rb_captured_block->code.iseq aliases with cfp->block_code. + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(block_iseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + specval: block_handler_specval, + write_block_code: false, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, cfunc_args); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result +} + /// Lowering for [`Insn::CCall`]. This is a low-level raw call that doesn't know /// anything about the callee, so handling for e.g. GC safety is dealt with elsewhere. -fn gen_ccall(asm: &mut Assembler, cfun: *const u8, args: Vec<Opnd>) -> lir::Opnd { - asm.ccall(cfun, args) +fn gen_ccall(asm: &mut Assembler, cfunc: *const u8, name: ID, recv: Opnd, args: Vec<Opnd>) -> lir::Opnd { + let mut cfunc_args = vec![recv]; + cfunc_args.extend(args); + asm.count_call_to(&name.contents_lossy()); + asm.ccall(cfunc, cfunc_args) +} + +// Change cfp->block_code in the current frame. See vm_caller_setup_arg_block(). +// VM_CFP_TO_CAPTURED_BLOCK then turns &cfp->self into a block handler. +// rb_captured_block->code.iseq aliases with cfp->block_code. +fn gen_block_handler_specval(asm: &mut Assembler, blockiseq: IseqPtr) -> lir::Opnd { + asm.store(Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_BLOCK_CODE), VALUE::from(blockiseq).into()); + let cfp_self_addr = asm.lea(Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_SELF)); + asm.or(cfp_self_addr, Opnd::Imm(1)) +} + +/// Generate code for a variadic C function call +/// func(int argc, VALUE *argv, VALUE recv) +fn gen_ccall_variadic( + jit: &mut JITState, + asm: &mut Assembler, + cfunc: *const u8, + name: ID, + recv: Opnd, + args: Vec<Opnd>, + cme: *const rb_callable_method_entry_t, + block: Option<BlockHandler>, + state: &FrameState, +) -> lir::Opnd { + gen_incr_counter(asm, Counter::variadic_cfunc_optimized_send_count); + gen_stack_overflow_check(jit, asm, state, state.stack_size()); + + let args_with_recv_len = args.len() + 1; + + // Compute the caller's stack size after consuming recv and args. + // state.stack() includes recv + args, so subtract both. + let caller_stack_size = state.stack_size() - args_with_recv_len; + + // Can't use gen_prepare_non_leaf_call() because we need to adjust the SP + // to account for the receiver and arguments (like gen_ccall_with_frame does) + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, caller_stack_size); + gen_spill_stack(jit, asm, state); + gen_spill_locals(jit, asm, state); + + let block_handler_specval = if let Some(BlockHandler::BlockIseq(blockiseq)) = block { + gen_block_handler_specval(asm, blockiseq) + } else { + VM_BLOCK_HANDLER_NONE.into() + }; + + gen_push_frame(asm, args_with_recv_len, state, ControlFrame { + recv, + iseq: None, + cme, + frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, + specval: block_handler_specval, + write_block_code: false, + }); + + asm_comment!(asm, "switch to new SP register"); + let sp_offset = (caller_stack_size + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; + let new_sp = asm.add(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + asm_comment!(asm, "switch to new CFP"); + let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let argv_ptr = gen_push_opnds(jit, asm, &args); + asm.count_call_to(&name.contents_lossy()); + let result = asm.ccall(cfunc, vec![args.len().into(), argv_ptr, recv]); + + asm_comment!(asm, "pop C frame"); + let new_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); + asm.mov(CFP, new_cfp); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + asm_comment!(asm, "restore SP register for the caller"); + let new_sp = asm.sub(SP, sp_offset.into()); + asm.mov(SP, new_sp); + + result } /// Emit an uncached instance variable lookup -fn gen_getivar(asm: &mut Assembler, recv: Opnd, id: ID) -> Opnd { - asm_ccall!(asm, rb_ivar_get, recv, id.0.into()) +fn gen_getivar(asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, state: &FrameState) -> Opnd { + if ic.is_null() { + asm_ccall!(asm, rb_ivar_get, recv, id.0.into()) + } else { + let iseq = Opnd::Value(state.iseq.into()); + asm_ccall!(asm, rb_vm_getinstancevariable, iseq, recv, id.0.into(), Opnd::const_ptr(ic)) + } } /// Emit an uncached instance variable store -fn gen_setivar(asm: &mut Assembler, recv: Opnd, id: ID, val: Opnd) { - asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); +fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { + // Setting an ivar can raise FrozenError, so we need proper frame state for exception handling. + gen_prepare_non_leaf_call(jit, asm, state); + if ic.is_null() { + asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); + } else { + let iseq = Opnd::Value(state.iseq.into()); + asm_ccall!(asm, rb_vm_setinstancevariable, iseq, recv, id.0.into(), val, Opnd::const_ptr(ic)); + } +} + +fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_getclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), Opnd::const_ptr(ic)) +} + +fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_setclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), val, Opnd::const_ptr(ic)); } /// Look up global variables -fn gen_getglobal(asm: &mut Assembler, id: ID) -> Opnd { +fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, state: &FrameState) -> Opnd { + // `Warning` module's method `warn` can be called when reading certain global variables + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_gvar_get, id.0.into()) } /// Intern a string fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Opnd { - gen_prepare_call_with_gc(asm, state); + gen_prepare_leaf_call_with_gc(asm, state); asm_ccall!(asm, rb_str_intern, val) } @@ -631,12 +1240,18 @@ fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, sta } /// Side-exit into the interpreter -fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, state: &FrameState) { - asm.jmp(side_exit(jit, state, *reason)); +fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, recompile: Option<Recompile>, state: &FrameState) { + asm.jmp(side_exit_with_recompile(jit, state, *reason, recompile)); } /// Emit a special object lookup -fn gen_putspecialobject(asm: &mut Assembler, value_type: SpecialObjectType) -> Opnd { +fn gen_putspecialobject(jit: &JITState, asm: &mut Assembler, value_type: SpecialObjectType, state: &FrameState) -> Opnd { + // rb_vm_get_special_object for CBASE/CONST_BASE can call rb_singleton_class, + // which allocates (may trigger GC) and can raise TypeError on non-class + // receivers (e.g. `123.instance_eval { Const = 1 }`). Treat as non-leaf so + // the PC is saved for GC and stack/locals are spilled for rescue. + gen_prepare_non_leaf_call(jit, asm, state); + // Get the EP of the current CFP and load it into a register let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); let ep_reg = asm.load(ep_opnd); @@ -644,9 +1259,12 @@ fn gen_putspecialobject(asm: &mut Assembler, value_type: SpecialObjectType) -> O asm_ccall!(asm, rb_vm_get_special_object, ep_reg, Opnd::UImm(u64::from(value_type))) } -fn gen_getspecial_symbol(asm: &mut Assembler, symbol_type: SpecialBackrefSymbol) -> Opnd { - // Fetch a "special" backref based on the symbol type +fn gen_getspecial_symbol(asm: &mut Assembler, symbol_type: SpecialBackrefSymbol, state: &FrameState) -> Opnd { + // rb_backref_get reaches rb_vm_svar_lep, which calls CFP_PC/CFP_ISEQ on the + // current frame, so the PC must be saved before the call. + gen_prepare_leaf_call_with_gc(asm, state); + // Fetch a "special" backref based on the symbol type let backref = asm_ccall!(asm, rb_backref_get,); match symbol_type { @@ -666,12 +1284,13 @@ fn gen_getspecial_symbol(asm: &mut Assembler, symbol_type: SpecialBackrefSymbol) } fn gen_getspecial_number(asm: &mut Assembler, nth: u64, state: &FrameState) -> Opnd { - // Fetch the N-th match from the last backref based on type shifted by 1 + // rb_backref_get reaches rb_vm_svar_lep, which calls CFP_PC/CFP_ISEQ on the + // current frame, so the PC must be saved before the call. + gen_prepare_leaf_call_with_gc(asm, state); + // Fetch the N-th match from the last backref based on type shifted by 1 let backref = asm_ccall!(asm, rb_backref_get,); - gen_prepare_call_with_gc(asm, state); - asm_ccall!(asm, rb_reg_nth_match, Opnd::Imm((nth >> 1).try_into().unwrap()), backref) } @@ -681,176 +1300,219 @@ fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameSt asm_comment!(asm, "RUBY_VM_CHECK_INTS(ec)"); // Not checking interrupt_mask since it's zero outside finalize_deferred_heap_pages, // signal_exec, or rb_postponed_job_flush. - let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG)); + let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG as i32)); asm.test(interrupt_flag, interrupt_flag); - asm.jnz(side_exit(jit, state, SideExitReason::Interrupt)); + asm.jnz(jit, side_exit(jit, state, SideExitReason::Interrupt)); } -/// Compile an interpreter entry block to be inserted into an ISEQ -fn gen_entry_prologue(asm: &mut Assembler, iseq: IseqPtr) { - asm_comment!(asm, "ZJIT entry point: {}", iseq_get_location(iseq, 0)); - // Save the registers we'll use for CFP, EP, SP - asm.frame_setup(lir::JIT_PRESERVED_REGS, 0); +fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_hash_resurrect, val) +} - // EC and CFP are passed as arguments - asm.mov(EC, C_ARG_OPNDS[0]); - asm.mov(CFP, C_ARG_OPNDS[1]); +fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aref, hash, key) +} - // Load the current SP from the CFP into REG_SP - asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); +fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_hash_aset, hash, key, val); +} - // TODO: Support entry chain guard when ISEQ has_opt -} - -/// Assign method arguments to basic block arguments at JIT entry -fn gen_entry_params(asm: &mut Assembler, iseq: IseqPtr, entry_block: &Block) { - let num_params = entry_block.params().len() - 1; // -1 to exclude self - if num_params > 0 { - asm_comment!(asm, "set method params: {num_params}"); - - // Fill basic block parameters. - // Doing it in reverse is load-bearing. High index params have memory slots that might - // require using a register to fill. Filling them first avoids clobbering. - for idx in (0..num_params).rev() { - let param = param_opnd(idx + 1); // +1 for self - let local = gen_entry_param(asm, iseq, idx); - - // Funky offset adjustment to write into the native stack frame of the - // HIR function we'll be calling into. This only makes sense in context - // of the schedule of instructions in gen_entry() for the JIT entry point. - // - // The entry point needs to load VALUEs into native stack slots _before_ the - // frame containing the slots exists. So, we anticipate the stack frame size - // of the Function and subtract offsets based on that. - // - // native SP at entry point ─────►┌────────────┐ Native SP grows downwards - // │ │ ↓ on all arches we support. - // SP-0x8 ├────────────┤ - // │ │ - // where native SP SP-0x10├────────────┤ - // would be while │ │ - // the HIR function ────────────► └────────────┘ - // is running - match param { - Opnd::Mem(lir::Mem { base: _, disp, num_bits }) => { - let param_slot = Opnd::mem(num_bits, NATIVE_STACK_PTR, disp - Assembler::frame_size()); - asm.mov(param_slot, local); - } - // Prepare for parallel move for locals in registers - reg @ Opnd::Reg(_) => { - asm.load_into(reg, local); - } - _ => unreachable!("on entry, params are either in memory or in reg. Got {param:?}") - } +fn gen_array_push(asm: &mut Assembler, array: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_push, array, val); +} - // Assign local variables to the basic block arguments - } - } - asm.load_into(param_opnd(SELF_PARAM_IDX), Opnd::mem(VALUE_BITS, CFP, RUBY_OFFSET_CFP_SELF)); +fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qtrue), val) } -/// Set branch params to basic block arguments -fn gen_branch_params(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { - if branch.args.is_empty() { - return; +fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qfalse), val) +} + +fn gen_defined_ivar(asm: &mut Assembler, self_val: Opnd, id: ID, pushval: VALUE) -> lir::Opnd { + asm_ccall!(asm, rb_zjit_defined_ivar, self_val, id.0.into(), Opnd::Value(pushval)) +} + +fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, target: Opnd, pattern: Opnd, flag: u32, state: &FrameState) -> lir::Opnd { + // rb_vm_check_match is not leaf unless flag is VM_CHECKMATCH_TYPE_WHEN. + // See also: leafness_of_checkmatch() and check_match() + if flag != VM_CHECKMATCH_TYPE_WHEN { + gen_prepare_non_leaf_call(jit, asm, state); } - asm_comment!(asm, "set branch params: {}", branch.args.len()); - let mut moves: Vec<(Reg, Opnd)> = vec![]; - for (idx, &arg) in branch.args.iter().enumerate() { - match param_opnd(idx) { - Opnd::Reg(reg) => { - // If a parameter is a register, we need to parallel-move it - moves.push((reg, jit.get_opnd(arg))); - }, - param => { - // If a parameter is memory, we set it beforehand - asm.mov(param, jit.get_opnd(arg)); - } - } + unsafe extern "C" { + fn rb_vm_check_match(ec: EcPtr, target: VALUE, pattern: VALUE, flag: u32) -> VALUE; } - asm.parallel_mov(moves); + + asm_ccall!(asm, rb_vm_check_match, EC, target, pattern, flag.into()) } -/// Get a method parameter on JIT entry. As of entry, whether EP is escaped or not solely -/// depends on the ISEQ type. -fn gen_entry_param(asm: &mut Assembler, iseq: IseqPtr, local_idx: usize) -> lir::Opnd { - let ep_offset = local_idx_to_ep_offset(iseq, local_idx); +fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_ary_concat, left, right); +} - // If the ISEQ does not escape EP, we can optimize the local variable access using the SP register. - if !iseq_entry_escapes_ep(iseq) { - // Create a reference to the local variable using the SP register. We assume EP == BP. - // TODO: Implement the invalidation in rb_zjit_invalidate_ep_is_bp() - let offs = -(SIZEOF_VALUE_I32 * (ep_offset + 1)); - Opnd::mem(64, SP, offs) - } else { - // Get the EP of the current CFP - let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); - let ep_reg = asm.load(ep_opnd); +fn gen_load_pc(asm: &mut Assembler) -> Opnd { + asm.load(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC)) +} + +fn gen_load_ec() -> Opnd { + EC +} + +fn gen_load_sp() -> Opnd { + SP +} + +fn gen_load_self() -> Opnd { + Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SELF) +} + +fn gen_load_field(asm: &mut Assembler, recv: Opnd, id: FieldName, offset: i32, return_type: Type) -> Opnd { + gen_incr_counter(asm, Counter::load_field_count); + asm_comment!(asm, "Load field id={id} offset={offset}"); + let recv = asm.load_mem(recv); + asm.load(Opnd::mem(return_type.num_bits(), recv, offset)) +} + +fn gen_store_field(asm: &mut Assembler, recv: Opnd, id: FieldName, offset: i32, val: Opnd, val_type: Type) { + gen_incr_counter(asm, Counter::store_field_count); + asm_comment!(asm, "Store field id={id} offset={offset}"); + let recv = asm.load_mem(recv); + asm.store(Opnd::mem(val_type.num_bits(), recv, offset), val); +} + +fn gen_write_barrier(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, val: Opnd, val_type: Type) { + // See RB_OBJ_WRITE/rb_obj_write: it's just assignment and rb_obj_written(). + // rb_obj_written() does: if (!RB_SPECIAL_CONST_P(val)) { rb_gc_writebarrier(recv, val); } + if !val_type.is_immediate() { + asm_comment!(asm, "Write barrier"); + let recv = asm.load_mem(recv); + + // Create a result block that all paths converge to + let hir_block_id = asm.current_block().hir_block_id; + let rpo_idx = asm.current_block().rpo_index; + let result_block = asm.new_block(hir_block_id, false, rpo_idx); + let result_edge = Target::Block(lir::BranchEdge { target: result_block, args: vec![] }); + + // If non-false immediate, don't fire write barrier + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(jit, result_edge.clone()); + + // If false, don't fire write barrier + asm.cmp(val, Qfalse.into()); + asm.je(jit, result_edge.clone()); + + // Heap object; fire the write barrier + asm_ccall!(asm, rb_gc_writebarrier, recv, val); + asm.jmp(result_edge); - // Create a reference to the local variable using cfp->ep - let offs = -(SIZEOF_VALUE_I32 * ep_offset); - Opnd::mem(64, ep_reg, offs) + // Join block + asm.set_current_block(result_block); + let label = jit.get_label(asm, result_block, hir_block_id); + asm.write_label(label); } } +/// Compile an interpreter entry block to be inserted into an ISEQ +fn gen_entry_prologue(asm: &mut Assembler) { + asm_comment!(asm, "ZJIT entry trampoline"); + // Save the registers we'll use for CFP, EP, SP + asm.frame_setup(lir::JIT_PRESERVED_REGS); + + // EC and CFP are passed as arguments + asm.mov(EC, C_ARG_OPNDS[0]); + asm.mov(CFP, C_ARG_OPNDS[1]); + + // Load the current SP from the CFP into REG_SP + asm.mov(SP, Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP)); +} + /// Compile a constant -fn gen_const(val: VALUE) -> lir::Opnd { +fn gen_const_value(val: VALUE) -> lir::Opnd { // Just propagate the constant value and generate nothing Opnd::Value(val) } -/// Compile a basic block argument -fn gen_param(asm: &mut Assembler, idx: usize) -> lir::Opnd { - // Allocate a register or a stack slot - match param_opnd(idx) { - // If it's a register, insert LiveReg instruction to reserve the register - // in the register pool for register allocation. - param @ Opnd::Reg(_) => asm.live_reg_opnd(param), - param => param, - } +/// Compile Const::CPtr +fn gen_const_cptr(val: *const u8) -> lir::Opnd { + Opnd::const_ptr(val) } -/// Compile a jump to a basic block -fn gen_jump(jit: &mut JITState, asm: &mut Assembler, branch: &BranchEdge) { - // Set basic block arguments - gen_branch_params(jit, asm, branch); +fn gen_const_long(val: i64) -> lir::Opnd { + Opnd::Imm(val) +} - // Jump to the basic block - let target = jit.get_label(asm, branch.target); - asm.jmp(target); +fn gen_const_uint16(val: u16) -> lir::Opnd { + Opnd::UImm(val as u64) } -/// Compile a conditional branch to a basic block -fn gen_if_true(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { - // If val is zero, move on to the next instruction. - let if_false = asm.new_label("if_false"); - asm.test(val, val); - asm.jz(if_false.clone()); +fn gen_const_uint32(val: u32) -> lir::Opnd { + Opnd::UImm(val as u64) +} - // If val is not zero, set basic block arguments and jump to the branch target. - // TODO: Consider generating the loads out-of-line - let if_true = jit.get_label(asm, branch.target); - gen_branch_params(jit, asm, branch); - asm.jmp(if_true); +fn gen_const_attr_index_t(val: attr_index_t) -> lir::Opnd { + Opnd::UImm(val as u64) +} - asm.write_label(if_false); +/// Compile a basic block argument +fn gen_param(asm: &mut Assembler, _idx: usize) -> lir::Opnd { + let vreg = asm.new_block_param(VALUE_BITS); + asm.current_block().add_parameter(vreg); + vreg } -/// Compile a conditional branch to a basic block -fn gen_if_false(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, branch: &BranchEdge) { - // If val is not zero, move on to the next instruction. - let if_true = asm.new_label("if_true"); - asm.test(val, val); - asm.jnz(if_true.clone()); +/// Compile a dynamic dispatch with block +fn gen_send( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); - // If val is zero, set basic block arguments and jump to the branch target. - // TODO: Consider generating the loads out-of-line - let if_false = jit.get_label(asm, branch.target); - gen_branch_params(jit, asm, branch); - asm.jmp(if_false); + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_send, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} - asm.write_label(if_true); +/// Compile a dynamic dispatch with `...` +fn gen_send_forward( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); + unsafe extern "C" { + fn rb_vm_sendforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_sendforward, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) } /// Compile a dynamic dispatch without block @@ -859,36 +1521,26 @@ fn gen_send_without_block( asm: &mut Assembler, cd: *const rb_call_data, state: &FrameState, + reason: SendFallbackReason, ) -> lir::Opnd { - // Note that it's incorrect to use this frame state to side exit because - // the state might not be on the boundary of an interpreter instruction. - // For example, `opt_aref_with` pushes to the stack and then sends. - asm_comment!(asm, "spill frame state"); - - // Save PC and SP - gen_save_pc(asm, state); - gen_save_sp(asm, state.stack().len()); - - // Spill locals and stack - gen_spill_locals(jit, asm, state); - gen_spill_stack(jit, asm, state); + gen_incr_send_fallback_counter(asm, reason); + gen_prepare_non_leaf_call(jit, asm, state); asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; } - let ret = asm.ccall( - rb_vm_opt_send_without_block as *const u8, - vec![EC, CFP, (cd as usize).into()], - ); - // TODO(max): Add a PatchPoint here that can side-exit the function if the callee messed with - // the frame's locals - - ret + asm_ccall!( + asm, + rb_vm_opt_send_without_block, + EC, CFP, Opnd::const_ptr(cd) + ) } -/// Compile a direct jump to an ISEQ call without block -fn gen_send_without_block_direct( +/// Compile a direct call to an ISEQ method. +/// If `block_handler` is provided, it's used as the specval for the new frame (for forwarding blocks). +/// Otherwise, `VM_BLOCK_HANDLER_NONE` is used. +fn gen_send_iseq_direct( cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, @@ -896,45 +1548,130 @@ fn gen_send_without_block_direct( iseq: IseqPtr, recv: Opnd, args: Vec<Opnd>, + kw_bits: u32, state: &FrameState, + block: Option<BlockHandler>, ) -> lir::Opnd { + gen_incr_counter(asm, Counter::iseq_optimized_send_count); + + let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); + let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); + gen_stack_overflow_check(jit, asm, state, stack_growth); + // Save cfp->pc and cfp->sp for the caller frame - gen_save_pc(asm, state); + // Can't use gen_prepare_non_leaf_call because we need special SP math. + gen_save_pc_for_gc(asm, state); gen_save_sp(asm, state.stack().len() - args.len() - 1); // -1 for receiver gen_spill_locals(jit, asm, state); gen_spill_stack(jit, asm, state); + // This mirrors vm_caller_setup_arg_block() in for the `blockiseq != NULL` case. + // The HIR specialization guards ensure we will only reach here for literal blocks, + // not &block forwarding, &:foo, etc. Thise are rejected in `type_specialize` by + // `unspecializable_call_type`. + let block_handler = block.map(|bh| match bh { BlockHandler::BlockIseq(b) => gen_block_handler_specval(asm, b), BlockHandler::BlockArg => unreachable!("BlockArg in gen_send_iseq_direct") }); + + let callee_is_bmethod = VM_METHOD_TYPE_BMETHOD == unsafe { get_cme_def_type(cme) }; + + let (frame_type, specval) = if callee_is_bmethod { + // Extract EP from the Proc instance + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let bmethod_frame_type = VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA; + // Tag the captured EP like VM_GUARDED_PREV_EP() in vm_call_iseq_bmethod() + let bmethod_specval = (capture.ep.addr() | 1).into(); + (bmethod_frame_type, bmethod_specval) + } else { + let specval = block_handler.unwrap_or_else(|| VM_BLOCK_HANDLER_NONE.into()); + (VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, specval) + }; + // Set up the new frame // TODO: Lazily materialize caller frames on side exits or when needed gen_push_frame(asm, args.len(), state, ControlFrame { recv, - iseq, + iseq: Some(iseq), cme, - frame_type: VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, + frame_type, + specval, + write_block_code: iseq_may_write_block_code(iseq), }); + // Write "keyword_bits" to the callee's frame if the callee accepts keywords. + // This is a synthetic local/parameter that the callee reads via checkkeyword to determine + // which optional keyword arguments need their defaults evaluated. + // We write this to the local table slot at bits_start so that: + // 1. The interpreter can read it via checkkeyword if we side-exit + // 2. The JIT entry can read it from the callee frame slot + if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + let bits_start = unsafe { (*keyword).bits_start } as usize; + let unspecified_bits = VALUE::fixnum_from_usize(kw_bits as usize); + let bits_offset = (state.stack().len() - args.len() + bits_start) * SIZEOF_VALUE; + asm_comment!(asm, "write keyword bits to callee frame"); + asm.store(Opnd::mem(64, SP, bits_offset as i32), unspecified_bits.into()); + } + asm_comment!(asm, "switch to new SP register"); - let local_size = unsafe { get_iseq_body_local_table_size(iseq) } as usize; - let sp_offset = (state.stack().len() + local_size - args.len() + VM_ENV_DATA_SIZE as usize) * SIZEOF_VALUE; + let sp_offset = (state.stack().len() + local_size - args.len() + VM_ENV_DATA_SIZE.to_usize()) * SIZEOF_VALUE; let new_sp = asm.add(SP, sp_offset.into()); asm.mov(SP, new_sp); asm_comment!(asm, "switch to new CFP"); let new_cfp = asm.sub(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, new_cfp); - asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.store(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); + + let params = unsafe { iseq.params() }; + + // For &block, the JIT entrypoint expects the block_handler as an argument + // This HIR param is not actually used, things read from specval from the VM frame today. + // TODO: Remove unused param from HIR, or pass specval through c_args. + // See https://github.com/ruby/ruby/pull/15911#discussion_r2710544982 + let needs_block = params.flags.has_block() != 0; // Set up arguments - let mut c_args = vec![recv]; - c_args.extend(args); + let mut c_args = Vec::with_capacity({ + // This is a heuristic to avoid re-allocation, not necessary for correctness + 1 /* recv */ + args.len() + if needs_block { 1 } else { 0 } + }); + c_args.push(recv); + c_args.extend(&args); + if needs_block { + if callee_is_bmethod { + // For bmethods, specval is the captured EP, not the block handler. + // The block param needs nil (no block) or a Proc value. + assert!(block_handler.is_none(), "at the moment, HIR builder never emits a direct send for a to-bmethod send-with-literal-block"); + c_args.push(Qnil.into()); + } else { + c_args.push(specval); + } + } + + let num_optionals_passed = if params.flags.has_opt() != 0 { + // See vm_call_iseq_setup_normal_opt_start in vm_inshelper.c + let lead_num = params.lead_num as u32; + let opt_num = params.opt_num as u32; + let post_num = params.post_num as u32; + let keyword = params.keyword; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } } as u32; + assert!(args.len() as u32 <= lead_num + opt_num + post_num + kw_total_num); + // For computing optional positional entry point, only count positional args + // and exclude the always-present lead and post slots. + let positional_argc = args.len() as u32 - kw_total_num; + let num_optionals_passed = positional_argc.saturating_sub(lead_num + post_num); + num_optionals_passed + } else { + 0 + }; // Make a method call. The target address will be rewritten once compiled. - let iseq_call = IseqCall::new(iseq); + let iseq_call = IseqCall::new(iseq, num_optionals_passed.try_into().expect("checked in HIR"), args.len().try_into().expect("checked in HIR")); let dummy_ptr = cb.get_write_ptr().raw_ptr(cb); jit.iseq_calls.push(iseq_call.clone()); - // TODO(max): Add a PatchPoint here that can side-exit the function if the callee messed with - // the frame's locals let ret = asm.ccall_with_iseq_call(dummy_ptr, c_args, &iseq_call); // If a callee side-exits, i.e. returns Qundef, propagate the return value to the caller. @@ -943,7 +1680,7 @@ fn gen_send_without_block_direct( asm_comment!(asm, "side-exit if callee side-exits"); asm.cmp(ret, Qundef.into()); // Restore the C stack pointer on exit - asm.je(ZJITState::get_exit_trampoline().into()); + asm.je(jit, ZJITState::get_exit_trampoline().into()); asm_comment!(asm, "restore SP register for the caller"); let new_sp = asm.sub(SP, sp_offset.into()); @@ -952,73 +1689,459 @@ fn gen_send_without_block_direct( ret } +/// Compile for invokeblock +fn gen_invokeblock( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeblock"); + unsafe extern "C" { + fn rb_vm_invokeblock(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokeblock, + EC, CFP, Opnd::const_ptr(cd) + ) +} + +/// Compile invokeblock for IFUNC block handlers. +/// Calls rb_vm_yield_with_cfunc directly. +fn gen_invokeblock_ifunc( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + block_handler: Opnd, + args: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + let _ = cd; // cd is not needed for the direct call + + gen_prepare_non_leaf_call(jit, asm, state); + + // Push args to memory so we can pass argv pointer + let argv_ptr = gen_push_opnds(jit, asm, &args); + + // Untag the block handler to get the captured block pointer + // captured = block_handler & ~0x3 + asm_comment!(asm, "untag block handler to get captured block"); + let captured = asm.and(block_handler, Opnd::Imm(!0x3i64)); + + asm_comment!(asm, "call rb_vm_yield_with_cfunc"); + unsafe extern "C" { + fn rb_vm_yield_with_cfunc( + ec: EcPtr, + captured: VALUE, + argc: i32, + argv: *const VALUE, + ) -> VALUE; + } + asm_ccall!(asm, rb_vm_yield_with_cfunc, EC, captured, (args.len() as i64).into(), argv_ptr) +} + +fn gen_invokeproc( + jit: &mut JITState, + asm: &mut Assembler, + recv: Opnd, + args: Vec<Opnd>, + kw_splat: bool, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + asm_comment!(asm, "call invokeproc"); + + let argv_ptr = gen_push_opnds(jit, asm, &args); + let kw_splat_opnd = Opnd::Imm(i64::from(kw_splat)); + asm_ccall!( + asm, + rb_optimized_call, + recv, + EC, + args.len().into(), + argv_ptr, + kw_splat_opnd, + VM_BLOCK_HANDLER_NONE.into() + ) +} + +/// Compile a dynamic dispatch for `super` +fn gen_invokesuper( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call super with dynamic dispatch"); + unsafe extern "C" { + fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokesuper, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + +/// Compile a dynamic dispatch for `super` with `...` +fn gen_invokesuperforward( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + blockiseq: IseqPtr, + state: &FrameState, + reason: SendFallbackReason, +) -> lir::Opnd { + gen_incr_send_fallback_counter(asm, reason); + + gen_prepare_non_leaf_call(jit, asm, state); + asm_comment!(asm, "call super with dynamic dispatch (forwarding)"); + unsafe extern "C" { + fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_invokesuperforward, + EC, CFP, Opnd::const_ptr(cd), VALUE::from(blockiseq).into() + ) +} + /// Compile a string resurrection fn gen_string_copy(asm: &mut Assembler, recv: Opnd, chilled: bool, state: &FrameState) -> Opnd { // TODO: split rb_ec_str_resurrect into separate functions - gen_prepare_call_with_gc(asm, state); + gen_prepare_leaf_call_with_gc(asm, state); let chilled = if chilled { Opnd::Imm(1) } else { Opnd::Imm(0) }; asm_ccall!(asm, rb_ec_str_resurrect, EC, recv, chilled) } +fn gen_string_equal(asm: &mut Assembler, left: Opnd, right: Opnd) -> lir::Opnd { + asm_ccall!(asm, rb_yarv_str_eql_internal, left, right) +} + /// Compile an array duplication instruction fn gen_array_dup( asm: &mut Assembler, val: lir::Opnd, state: &FrameState, ) -> lir::Opnd { - gen_prepare_call_with_gc(asm, state); + gen_prepare_leaf_call_with_gc(asm, state); asm_ccall!(asm, rb_ary_resurrect, val) } /// Compile a new array instruction fn gen_new_array( + jit: &JITState, asm: &mut Assembler, elements: Vec<Opnd>, state: &FrameState, ) -> lir::Opnd { - gen_prepare_call_with_gc(asm, state); + gen_prepare_leaf_call_with_gc(asm, state); - let length: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + let num: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); - let new_array = asm_ccall!(asm, rb_ary_new_capa, length.into()); + if elements.is_empty() { + asm_ccall!(asm, rb_ec_ary_new_from_values, EC, 0i64.into(), Opnd::UImm(0)) + } else { + let argv = gen_push_opnds(jit, asm, &elements); + asm_ccall!(asm, rb_ec_ary_new_from_values, EC, num.into(), argv) + } +} - for val in elements { - asm_ccall!(asm, rb_ary_push, new_array, val); +/// Adjust potentially-negative index by the given length, returning the adjusted index. If still negative, +/// return a negative number, which indicates the index is still out-of-bounds. +fn gen_adjust_bounds(asm: &mut Assembler, index: Opnd, length: Opnd) -> lir::Opnd { + let adjusted = asm.add(index, length); + asm.test(index, index); + asm.csel_l(adjusted, index) +} + +/// Compile array access (`array[index]`) +fn gen_array_aref( + asm: &mut Assembler, + array: Opnd, + index: Opnd, +) -> lir::Opnd { + let unboxed_idx = asm.load_mem(index); + let array = asm.load_mem(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.load(Opnd::mem(VALUE_BITS, elem_ptr, 0)) +} + +fn gen_array_aset( + asm: &mut Assembler, + array: Opnd, + index: Opnd, + val: Opnd, +) { + let unboxed_idx = asm.load_mem(index); + let array = asm.load_mem(array); + let array_ptr = gen_array_ptr(asm, array); + let elem_offset = asm.lshift(unboxed_idx, Opnd::UImm(SIZEOF_VALUE.trailing_zeros() as u64)); + let elem_ptr = asm.add(array_ptr, elem_offset); + asm.store(Opnd::mem(VALUE_BITS, elem_ptr, 0), val); +} + +fn gen_array_pop(asm: &mut Assembler, array: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_ary_pop, array) +} + +fn gen_array_length(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let array = asm.load_mem(array); + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + let embedded_len = asm.and(flags, (RARRAY_EMBED_LEN_MASK as u64).into()); + let embedded_len = asm.rshift(embedded_len, (RARRAY_EMBED_LEN_SHIFT as u64).into()); + // cmov between the embedded length and heap length depending on the embed flag + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_len = Opnd::mem(c_long::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_LEN); + asm.csel_nz(embedded_len, heap_len) +} + +fn gen_array_ptr(asm: &mut Assembler, array: Opnd) -> lir::Opnd { + let flags = Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RARRAY_EMBED_FLAG as u64).into()); + let heap_ptr = Opnd::mem(usize::BITS as u8, array, RUBY_OFFSET_RARRAY_AS_HEAP_PTR); + let embedded_ptr = asm.lea(Opnd::mem(VALUE_BITS, array, RUBY_OFFSET_RARRAY_AS_ARY)); + asm.csel_nz(embedded_ptr, heap_ptr) +} + +/// Compile opt_newarray_hash - create a hash from array elements +fn gen_opt_newarray_hash( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + // `Array#hash` will hash the elements of the array. + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_hash(ec: EcPtr, array_len: u32, elts: *const VALUE) -> VALUE; } - new_array + asm.ccall( + rb_vm_opt_newarray_hash as *const u8, + vec![EC, (array_len as u32).into(), elements_ptr], + ) } -/// Compile a new hash instruction -fn gen_new_hash( - jit: &mut JITState, +/// Compile ArrayMax - find the maximum element among array elements +fn gen_array_max( + jit: &JITState, asm: &mut Assembler, - elements: &Vec<(InsnId, InsnId)>, + elements: Vec<Opnd>, state: &FrameState, ) -> lir::Opnd { gen_prepare_non_leaf_call(jit, asm, state); - let cap: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); - let new_hash = asm_ccall!(asm, rb_hash_new_with_size, lir::Opnd::Imm(cap)); + let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); - if !elements.is_empty() { - let mut pairs = Vec::new(); - for (key_id, val_id) in elements.iter() { - let key = jit.get_opnd(*key_id); - let val = jit.get_opnd(*val_id); - pairs.push(key); - pairs.push(val); - } + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(VALUE_BITS, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_max(ec: EcPtr, num: u32, elts: *const VALUE) -> VALUE; + } + + asm.ccall( + rb_vm_opt_newarray_max as *const u8, + vec![EC, array_len.into(), elements_ptr], + ) +} + +/// Find the minimum element among array elements +fn gen_array_min( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); - let argv = gen_push_opnds(jit, asm, &pairs); - let argc = (elements.len() * 2) as ::std::os::raw::c_long; - asm_ccall!(asm, rb_hash_bulk_insert, lir::Opnd::Imm(argc), argv, new_hash); + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len(); + let elements_ptr = asm.lea(Opnd::mem(VALUE_BITS, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); - gen_pop_opnds(asm, &pairs); + unsafe extern "C" { + fn rb_vm_opt_newarray_min(ec: EcPtr, num: u32, elts: *const VALUE) -> VALUE; } - new_hash + asm.ccall( + rb_vm_opt_newarray_min as *const u8, + vec![EC, array_len.into(), elements_ptr], + ) +} + +fn gen_array_include( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the target. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = state.stack().len() - elements.len() - 1; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_include_p(ec: EcPtr, num: c_long, elts: *const VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_include_p, + EC, array_len.into(), elements_ptr, target + ) +} + +fn gen_array_pack_buffer( + jit: &JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + fmt: Opnd, + buffer: Option<Opnd>, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); + + // After gen_prepare_non_leaf_call, the elements are spilled to the Ruby stack. + // The elements are at the bottom of the virtual stack, followed by the fmt, and optionally the buffer. + // Get a pointer to the first element on the Ruby stack. + let stack_bottom = if buffer.is_some() { + state.stack().len() - elements.len() - 2 + } else { + state.stack().len() - elements.len() - 1 + }; + let elements_ptr = asm.lea(Opnd::mem(64, SP, stack_bottom as i32 * SIZEOF_VALUE_I32)); + + unsafe extern "C" { + fn rb_vm_opt_newarray_pack_buffer(ec: EcPtr, num: c_long, elts: *const VALUE, fmt: VALUE, buffer: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_newarray_pack_buffer, + EC, array_len.into(), elements_ptr, fmt, buffer.unwrap_or_else(|| Qundef.into()) + ) +} + +fn gen_dup_array_include( + jit: &JITState, + asm: &mut Assembler, + ary: VALUE, + target: Opnd, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + + unsafe extern "C" { + fn rb_vm_opt_duparray_include_p(ec: EcPtr, ary: VALUE, target: VALUE) -> VALUE; + } + asm_ccall!( + asm, + rb_vm_opt_duparray_include_p, + EC, ary.into(), target + ) +} + +fn gen_is_a(jit: &mut JITState, asm: &mut Assembler, obj: Opnd, class: Opnd) -> lir::Opnd { + let builtin_type = match class { + Opnd::Value(value) if value == unsafe { rb_cString } => Some(RUBY_T_STRING), + Opnd::Value(value) if value == unsafe { rb_cArray } => Some(RUBY_T_ARRAY), + Opnd::Value(value) if value == unsafe { rb_cHash } => Some(RUBY_T_HASH), + _ => None + }; + + if let Some(builtin_type) = builtin_type { + asm_comment!(asm, "IsA by matching builtin type"); + let hir_block_id = asm.current_block().hir_block_id; + let rpo_idx = asm.current_block().rpo_index; + + // Create a result block that all paths converge to + let result_block = asm.new_block(hir_block_id, false, rpo_idx); + let result_edge = |v| Target::Block(lir::BranchEdge { + target: result_block, + args: vec![v], + }); + + let val = asm.load_mem(obj); + + // Immediate -> definitely not String/Array/Hash + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(jit, result_edge(Qfalse.into())); + + // Qfalse -> definitely not String/Array/Hash + asm.cmp(val, Qfalse.into()); + asm.je(jit, result_edge(Qfalse.into())); + + // Heap object -> check builtin type + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let obj_builtin_type = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(obj_builtin_type, Opnd::UImm(builtin_type as u64)); + let result = asm.csel_e(Qtrue.into(), Qfalse.into()); + asm.jmp(result_edge(result)); + + // Result block -- receives the value via block parameter (phi node) + asm.set_current_block(result_block); + let label = jit.get_label(asm, result_block, hir_block_id); + asm.write_label(label); + let param = asm.new_block_param(VALUE_BITS); + asm.current_block().add_parameter(param); + param + } else { + asm_ccall!(asm, rb_obj_is_kind_of, obj, class) + } +} + +/// Compile a new hash instruction +fn gen_new_hash( + jit: &mut JITState, + asm: &mut Assembler, + elements: Vec<Opnd>, + state: &FrameState, +) -> lir::Opnd { + if elements.is_empty() { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_hash_new,) + } else { + gen_prepare_non_leaf_call(jit, asm, state); + + let argv = gen_push_opnds(jit, asm, &elements); + asm_ccall!(asm, rb_hash_new_with_bulk_insert, elements.len().into(), argv) + } } /// Compile a new range instruction @@ -1034,9 +2157,73 @@ fn gen_new_range( gen_prepare_non_leaf_call(jit, asm, state); // Call rb_range_new(low, high, flag) + asm_ccall!(asm, rb_range_new, low, high, (flag as i32).into()) +} + +fn gen_new_range_fixnum( + asm: &mut Assembler, + low: lir::Opnd, + high: lir::Opnd, + flag: RangeType, + state: &FrameState, +) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); asm_ccall!(asm, rb_range_new, low, high, (flag as i64).into()) } +fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Allocating an object from an unknown class is non-leaf; see doc for `ObjectAlloc`. + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_obj_alloc, val) +} + +fn gen_object_alloc_class(asm: &mut Assembler, class: VALUE, state: &FrameState) -> lir::Opnd { + // Allocating an object for a known class with default allocator is leaf; see doc for + // `ObjectAllocClass`. + gen_prepare_leaf_call_with_gc(asm, state); + if unsafe { rb_zjit_class_has_default_allocator(class) } { + // TODO(max): inline code to allocate an instance + asm_ccall!(asm, rb_class_allocate_instance, class.into()) + } else { + assert!(class_has_leaf_allocator(class), "class passed to ObjectAllocClass must have a leaf allocator"); + let alloc_func = unsafe { rb_zjit_class_get_alloc_func(class) }; + assert!(alloc_func.is_some(), "class {} passed to ObjectAllocClass must have an allocator", get_class_name(class)); + asm_comment!(asm, "call allocator for class {}", get_class_name(class)); + asm.count_call_to(&format!("{}::allocator", get_class_name(class))); + asm.ccall(alloc_func.unwrap() as *const u8, vec![class.into()]) + } +} + +/// Map an entry point to the bytecode PC used by its initial JITFrame. +/// JIT call entries use `opt_table[jit_entry_idx]`; the interpreter entry uses +/// `opt_table.last()` for the fall-through path where all optionals are filled. +fn entry_pc(iseq: IseqPtr, jit_entry_idx: Option<usize>) -> *const VALUE { + let params = unsafe { iseq.params() }; + let opt_table = params.opt_table_slice(); + let entry_idx = jit_entry_idx.unwrap_or_else(|| opt_table.len() - 1); + let entry_insn_idx = opt_table.get(entry_idx) + .unwrap_or_else(|| panic!("entry_pc: opt_table out of bounds. {params:#?}, entry_idx={entry_idx}")) + .as_u32(); + unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idx) } +} + +/// Compile a frame setup. If jit_entry_idx is Some, remember the address of it as a JIT entry. +fn gen_entry_point(jit: &mut JITState, asm: &mut Assembler, jit_entry_idx: Option<usize>) { + if let Some(jit_entry_idx) = jit_entry_idx { + let jit_entry = JITEntry::new(jit_entry_idx); + jit.jit_entries.push(jit_entry.clone()); + asm.pos_marker(move |code_ptr, _| { + jit_entry.borrow_mut().start_addr.set(Some(code_ptr)); + }); + } + asm.frame_setup(&[]); + + // Publish a valid entry JITFrame before setting cfp->jit_return. + let jit_frame = JITFrame::new_iseq(entry_pc(jit.iseq(), jit_entry_idx), jit.iseq()); + asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, -SIZEOF_VALUE_I32), Opnd::const_ptr(jit_frame)); + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), NATIVE_BASE_PTR); +} + /// Compile code that exits from JIT code with a return value fn gen_return(asm: &mut Assembler, val: lir::Opnd) { // Pop the current frame (ec->cfp++) @@ -1044,14 +2231,14 @@ fn gen_return(asm: &mut Assembler, val: lir::Opnd) { asm_comment!(asm, "pop stack frame"); let incr_cfp = asm.add(CFP, RUBY_SIZEOF_CONTROL_FRAME.into()); asm.mov(CFP, incr_cfp); - asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP), CFP); + asm.mov(Opnd::mem(64, EC, RUBY_OFFSET_EC_CFP as i32), CFP); // Order here is important. Because we're about to tear down the frame, // we need to load the return value, which might be part of the frame. asm.load_into(C_RET_OPND, val); // Return from the function - asm.frame_teardown(&[]); // matching the setup in :bb0-prologue: + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() asm.cret(C_RET_OPND); } @@ -1060,7 +2247,7 @@ fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, righ // Add left + right and test for overflow let left_untag = asm.sub(left, Opnd::Imm(1)); let out_val = asm.add(left_untag, right); - asm.jo(side_exit(jit, state, FixnumAddOverflow)); + asm.jo(jit, side_exit(jit, state, FixnumAddOverflow)); out_val } @@ -1069,7 +2256,7 @@ fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, righ fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Subtract left - right and test for overflow let val_untag = asm.sub(left, right); - asm.jo(side_exit(jit, state, FixnumSubOverflow)); + asm.jo(jit, side_exit(jit, state, FixnumSubOverflow)); asm.add(val_untag, Opnd::Imm(1)) } @@ -1082,10 +2269,50 @@ fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, rig let out_val = asm.mul(left_untag, right_untag); // Test for overflow - asm.jo_mul(side_exit(jit, state, FixnumMultOverflow)); + asm.jo_mul(jit, side_exit(jit, state, FixnumMultOverflow)); asm.add(out_val, Opnd::UImm(1)) } +/// Compile Fixnum / Fixnum +fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + + // Side exit if rhs is 0 + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(jit, side_exit(jit, state, FixnumDivByZero)); + asm_ccall!(asm, rb_jit_fix_div_fix, left, right) +} + +/// Compile Float + Float +fn gen_float_add(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_plus, recv, other) +} + +/// Compile Float - Float +fn gen_float_sub(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_minus, recv, other) +} + +/// Compile Float * Float +fn gen_float_mul(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_mul, recv, other) +} + +/// Compile Float / Float +fn gen_float_div(asm: &mut Assembler, recv: lir::Opnd, other: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_float_div, recv, other) +} + +/// Compile Float#to_i (truncate to integer) +fn gen_float_to_int(asm: &mut Assembler, recv: lir::Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_leaf_call_with_gc(asm, state); + asm_ccall!(asm, rb_flo_to_i, recv) +} + /// Compile Fixnum == Fixnum fn gen_fixnum_eq(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { asm.cmp(left, right); @@ -1132,15 +2359,84 @@ fn gen_fixnum_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir: asm.or(left, right) } -// Compile val == nil -fn gen_isnil(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { - asm.cmp(val, Qnil.into()); - // TODO: Implement and use setcc +/// Compile C integer | C integer. +fn gen_int_or(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.or(left, right) +} + +/// Compile Fixnum ^ Fixnum +fn gen_fixnum_xor(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + // XOR and then re-tag the resulting fixnum + let out_val = asm.xor(left, right); + asm.add(out_val, Opnd::UImm(1)) +} + +/// Compile Fixnum << Fixnum +fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let in_val = asm.sub(left, Opnd::UImm(1)); // Drop tag bit + let out_val = asm.lshift(in_val, shift_amount.into()); + let unshifted = asm.rshift(out_val, shift_amount.into()); + asm.cmp(in_val, unshifted); + asm.jne(jit, side_exit(jit, state, FixnumLShiftOverflow)); + // Re-tag the output value + let out_val = asm.add(out_val, 1.into()); + out_val +} + +/// Compile Fixnum >> Fixnum +fn gen_fixnum_rshift(asm: &mut Assembler, left: lir::Opnd, shift_amount: u64) -> lir::Opnd { + // Shift amount is known statically to be in the range [0, 63] + assert!(shift_amount < 64); + let result = asm.rshift(left, shift_amount.into()); + // Re-tag the output value + asm.or(result, 1.into()) +} + +fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Check for left % 0, which raises ZeroDivisionError + asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); + asm.je(jit, side_exit(jit, state, FixnumModByZero)); + asm_ccall!(asm, rb_fix_mod_fix, left, right) +} + +fn gen_fixnum_aref(asm: &mut Assembler, recv: lir::Opnd, index: lir::Opnd) -> lir::Opnd { + asm_ccall!(asm, rb_fix_aref, recv, index) +} + +fn gen_is_method_cfunc(asm: &mut Assembler, val: lir::Opnd, cd: *const rb_call_data, cfunc: *const u8, state: &FrameState) -> lir::Opnd { + unsafe extern "C" { + fn rb_vm_method_cfunc_is(iseq: IseqPtr, cd: *const rb_call_data, recv: VALUE, cfunc: *const u8) -> VALUE; + } + asm_ccall!(asm, rb_vm_method_cfunc_is, VALUE::from(state.iseq).into(), Opnd::const_ptr(cd), val, Opnd::const_ptr(cfunc)) +} + +fn gen_is_bit_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) } +fn gen_is_bit_not_equal(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir::Opnd { + asm.cmp(left, right); + asm.csel_ne(Opnd::Imm(1), Opnd::Imm(0)) +} + +fn gen_box_bool(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { + asm.test(val, val); + asm.csel_nz(Opnd::Value(Qtrue), Opnd::Value(Qfalse)) +} + +fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { + // Load the value, then test for overflow and tag it + let val = asm.load_mem(val); + let shifted = asm.lshift(val, Opnd::UImm(1)); + asm.jo(jit, side_exit(jit, state, BoxFixnumOverflow)); + asm.or(shifted, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)) +} + fn gen_anytostring(asm: &mut Assembler, val: lir::Opnd, str: lir::Opnd, state: &FrameState) -> lir::Opnd { - gen_prepare_call_with_gc(asm, state); + gen_prepare_leaf_call_with_gc(asm, state); asm_ccall!(asm, rb_obj_as_string_result, str, val) } @@ -1156,31 +2452,106 @@ fn gen_test(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { asm.csel_e(0.into(), 1.into()) } +fn gen_has_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_type: Type, ty: Type) -> lir::Opnd { + if ty.is_subtype(types::Fixnum) { + asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + asm.csel_nz(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::Flonum) { + // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG + let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); + asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::StaticSymbol) { + // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG + // Use 8-bit comparison like YJIT does. + // If `val` is a constant (rare but possible), put it in a register to allow masking. + let val = asm.load_imm(val); + asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::NilClass) { + asm.cmp(val, Qnil.into()); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::TrueClass) { + asm.cmp(val, Qtrue.into()); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_subtype(types::FalseClass) { + asm.cmp(val, Qfalse.into()); + asm.csel_e(Opnd::Imm(1), Opnd::Imm(0)) + } else if ty.is_immediate() { + // All immediate types' guard should have been handled above + panic!("unexpected immediate guard type: {ty}"); + } else if let Some(expected_class) = ty.runtime_exact_ruby_class() { + let hir_block_id = asm.current_block().hir_block_id; + let rpo_idx = asm.current_block().rpo_index; + + // Create a result block that all paths converge to + let result_block = asm.new_block(hir_block_id, false, rpo_idx); + let result_edge = |v| Target::Block(lir::BranchEdge { + target: result_block, + args: vec![v], + }); + + // If val isn't in a register, load it to use it as the base of Opnd::mem later. + // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) + let val = asm.load_mem(val); + + let is_known_heap_basic_object = val_type.is_subtype(types::HeapBasicObject); + if !is_known_heap_basic_object { + // Immediate -> definitely not the class + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(jit, result_edge(Opnd::Imm(0))); + + // Qfalse -> definitely not the class + asm.cmp(val, Qfalse.into()); + asm.je(jit, result_edge(Opnd::Imm(0))); + } + + // Heap object -> check klass field + let klass = asm.load(Opnd::mem(64, val, RUBY_OFFSET_RBASIC_KLASS)); + asm.cmp(klass, Opnd::Value(expected_class)); + let result = asm.csel_e(Opnd::UImm(1), Opnd::Imm(0)); + asm.jmp(result_edge(result)); + + // Result block -- receives the value via block parameter (phi node) + asm.set_current_block(result_block); + let label = jit.get_label(asm, result_block, hir_block_id); + asm.write_label(label); + let param = asm.new_block_param(VALUE_BITS); + asm.current_block().add_parameter(param); + param + } else { + unimplemented!("unsupported type: {ty}"); + } +} + /// Compile a type check with a side exit -fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard_type: Type, state: &FrameState) -> lir::Opnd { +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_type: Type, guard_type: Type, recompile: Option<Recompile>, state: &FrameState) -> lir::Opnd { + let is_known_heap_basic_object = val_type.is_subtype(types::HeapBasicObject); + gen_incr_counter(asm, Counter::guard_type_count); if guard_type.is_subtype(types::Fixnum) { asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); - asm.jz(side_exit(jit, state, GuardType(guard_type))); + asm.jz(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::Flonum) { // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); - asm.jne(side_exit(jit, state, GuardType(guard_type))); + asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::StaticSymbol) { // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG - // Use 8-bit comparison like YJIT does. GuardType should not be used - // for a known VALUE, which with_num_bits() does not support. + // Use 8-bit comparison like YJIT does. + // If `val` is a constant (rare but possible), put it in a register to allow masking. + let val = asm.load_imm(val); asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); - asm.jne(side_exit(jit, state, GuardType(guard_type))); + asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::NilClass) { asm.cmp(val, Qnil.into()); - asm.jne(side_exit(jit, state, GuardType(guard_type))); + asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::TrueClass) { asm.cmp(val, Qtrue.into()); - asm.jne(side_exit(jit, state, GuardType(guard_type))); + asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::FalseClass) { asm.cmp(val, Qfalse.into()); - asm.jne(side_exit(jit, state, GuardType(guard_type))); + asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); } else if guard_type.is_immediate() { // All immediate types' guard should have been handled above panic!("unexpected immediate guard type: {guard_type}"); @@ -1189,25 +2560,49 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard // If val isn't in a register, load it to use it as the base of Opnd::mem later. // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) - let val = match val { - Opnd::Reg(_) | Opnd::VReg { .. } => val, - _ => asm.load(val), - }; + let val = asm.load_mem(val); - // Check if it's a special constant - let side_exit = side_exit(jit, state, GuardType(guard_type)); - asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); - asm.jnz(side_exit.clone()); + let side_exit = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + if !is_known_heap_basic_object { + // Check if it's a special constant + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(jit, side_exit.clone()); - // Check if it's false - asm.cmp(val, Qfalse.into()); - asm.je(side_exit.clone()); + // Check if it's false + asm.cmp(val, Qfalse.into()); + asm.je(jit, side_exit.clone()); + } // Load the class from the object's klass field let klass = asm.load(Opnd::mem(64, val, RUBY_OFFSET_RBASIC_KLASS)); asm.cmp(klass, Opnd::Value(expected_class)); - asm.jne(side_exit); + asm.jne(jit, side_exit); + } else if let Some(builtin_type) = guard_type.builtin_type_equivalent() { + let side = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + + if !is_known_heap_basic_object { + // Check special constant + asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64)); + asm.jnz(jit, side.clone()); + + // Check false + asm.cmp(val, Qfalse.into()); + asm.je(jit, side.clone()); + } + + // Mask and check the builtin type + let val = asm.load_mem(val); + let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS)); + let tag = asm.and(flags, Opnd::UImm(RUBY_T_MASK as u64)); + asm.cmp(tag, Opnd::UImm(builtin_type as u64)); + asm.jne(jit, side); + } else if guard_type.bit_equal(types::HeapBasicObject) { + let side_exit = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + asm.cmp(val, Opnd::Value(Qfalse)); + asm.je(jit, side_exit.clone()); + asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); + asm.jnz(jit, side_exit); } else { unimplemented!("unsupported type: {guard_type}"); } @@ -1215,30 +2610,167 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard } /// Compile an identity check with a side exit -fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: VALUE, state: &FrameState) -> lir::Opnd { - asm.cmp(val, Opnd::Value(expected)); - asm.jnz(side_exit(jit, state, GuardBitEquals(expected))); +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, recompile: Option<Recompile>, state: &FrameState) -> lir::Opnd { + if matches!(reason, SideExitReason::GuardShape(_) ) { + gen_incr_counter(asm, Counter::guard_shape_count); + } + let expected_opnd: Opnd = match expected { + crate::hir::Const::Value(v) => { Opnd::Value(v) } + crate::hir::Const::CInt64(v) => { v.into() } + crate::hir::Const::CShape(v) => { Opnd::UImm(v.0 as u64) } + _ => panic!("gen_guard_bit_equals: unexpected hir::Const {expected:?}"), + }; + asm.cmp(val, expected_opnd); + asm.jnz(jit, side_exit_with_recompile(jit, state, reason, recompile)); val } -/// Generate code that increments a counter in ZJIT stats +fn mask_to_opnd(mask: crate::hir::Const) -> Option<Opnd> { + match mask { + crate::hir::Const::CUInt8(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt16(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt32(v) => Some(Opnd::UImm(v as u64)), + crate::hir::Const::CUInt64(v) => Some(Opnd::UImm(v)), + _ => None + } +} + +/// Compile a bitmask check with a side exit if none of the masked bits are not set +fn gen_guard_any_bit_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_any_bit_set: unexpected hir::Const {mask:?}")); + asm.test(val, mask_opnd); + asm.jz(jit, side_exit(jit, state, reason)); + val +} + +/// Compile a bitmask check with a side exit if any of the masked bits are set +fn gen_guard_no_bits_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { + let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_no_bits_set: unexpected hir::Const {mask:?}")); + asm.test(val, mask_opnd); + asm.jnz(jit, side_exit(jit, state, reason)); + val +} + +/// Generate code that records unoptimized C functions if --zjit-stats is enabled +fn gen_incr_counter_ptr(asm: &mut Assembler, counter_ptr: *mut u64) { + if get_option!(stats) { + asm.incr_counter(Opnd::const_ptr(counter_ptr as *const u8), Opnd::UImm(1)); + } +} + +/// Generate code that increments a counter if --zjit-stats fn gen_incr_counter(asm: &mut Assembler, counter: Counter) { - let ptr = counter_ptr(counter); - let ptr_reg = asm.load(Opnd::const_ptr(ptr as *const u8)); - let counter_opnd = Opnd::mem(64, ptr_reg, 0); + if get_option!(stats) { + let ptr = counter_ptr(counter); + gen_incr_counter_ptr(asm, ptr); + } +} + +/// Increment a counter for each DynamicSendReason. If the variant has +/// a counter prefix to break down the details, increment that as well. +fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReason) { + gen_incr_counter(asm, send_fallback_counter(reason)); + + use SendFallbackReason::*; + match reason { + Uncategorized(opcode) => { + gen_incr_counter_ptr(asm, send_fallback_counter_ptr_for_opcode(opcode)); + } + SendWithoutBlockNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_method_type(method_type)); + } + SendWithoutBlockNotOptimizedMethodTypeOptimized(method_type) => { + gen_incr_counter(asm, send_without_block_fallback_counter_for_optimized_method_type(method_type)); + } + SendNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_method_type(method_type)); + } + SuperNotOptimizedMethodType(method_type) => { + gen_incr_counter(asm, send_fallback_counter_for_super_method_type(method_type)); + } + _ => {} + } +} + +/// Check if an ISEQ contains instructions that may write to block_code +/// (send, sendforward, invokesuper, invokesuperforward, invokeblock). +/// These instructions call vm_caller_setup_arg_block which writes to cfp->block_code. +#[allow(non_upper_case_globals)] +pub(crate) fn iseq_may_write_block_code(iseq: IseqPtr) -> bool { + let encoded_size = unsafe { rb_iseq_encoded_size(iseq) }; + let mut insn_idx: u32 = 0; + + while insn_idx < encoded_size { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + let opcode = unsafe { rb_iseq_bare_opcode_at_pc(iseq, pc) } as u32; + + match opcode { + YARVINSN_send | YARVINSN_sendforward | + YARVINSN_invokesuper | YARVINSN_invokesuperforward | + YARVINSN_invokeblock => { + return true; + } + _ => {} + } + + insn_idx = insn_idx.saturating_add(unsafe { rb_insn_len(VALUE(opcode as usize)) }.try_into().unwrap()); + } - // Increment and store the updated value - asm.incr_counter(counter_opnd, Opnd::UImm(1)); + false } -/// Save the incremented PC on the CFP. -/// This is necessary when callees can raise or allocate. -fn gen_save_pc(asm: &mut Assembler, state: &FrameState) { +/// Save only the PC to CFP. Use this when you need to call gen_save_sp() +/// immediately after with a custom stack size (e.g., gen_ccall_with_frame +/// adjusts SP to exclude receiver and arguments). +fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { let opcode: usize = state.get_opcode().try_into().unwrap(); let next_pc: *const VALUE = unsafe { state.pc.offset(insn_len(opcode) as isize) }; - asm_comment!(asm, "save PC to CFP"); - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(next_pc)); + gen_incr_counter(asm, Counter::vm_write_jit_frame_count); + asm_comment!(asm, "save JITFrame to CFP"); + let jit_frame = JITFrame::new_iseq(next_pc, state.iseq); + asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, -SIZEOF_VALUE_I32), Opnd::const_ptr(jit_frame)); + + // CFP_PC for a live JIT frame routes through the JITFrame on the native + // stack (cfp->jit_return points to NATIVE_BASE_PTR), so we don't need to + // touch cfp->pc here. Poisoning cfp->pc with PC_POISON would actively + // break the case where rb_zjit_materialize_frames() previously copied + // jit_frame->pc into cfp->pc and cleared cfp->jit_return: the JIT keeps + // running, lands on this routine again, and the poison would replace + // the valid materialized pc behind the GC's back. +} + +/// Save the current PC on the CFP as a preparation for calling a C function +/// that may allocate objects and trigger GC. Use gen_prepare_non_leaf_call() +/// if it may raise exceptions or call arbitrary methods. +/// +/// Unlike YJIT, we don't need to save the stack slots to protect them from GC +/// because the backend spills all live registers onto the C stack on CCall. +/// However, to avoid marking uninitialized stack slots, this also updates SP, +/// which may have cfp->sp for a past frame or a past non-leaf call. +fn gen_prepare_call_with_gc(asm: &mut Assembler, state: &FrameState, leaf: bool) { + gen_save_pc_for_gc(asm, state); + gen_save_sp(asm, state.stack_size()); + if leaf { + asm.expect_leaf_ccall(state.stack_size()); + } +} + +fn gen_prepare_leaf_call_with_gc(asm: &mut Assembler, state: &FrameState) { + // In gen_prepare_call_with_gc(), we update cfp->sp for leaf calls too. + // + // Here, cfp->sp may be pointing to either of the following: + // 1. cfp->sp for a past frame, which gen_push_frame() skips to initialize + // 2. cfp->sp set by gen_prepare_non_leaf_call() for the current frame + // + // When (1), to avoid marking dead objects, we need to set cfp->sp for the current frame. + // When (2), setting cfp->sp at gen_push_frame() and not updating cfp->sp here could lead to + // keeping objects longer than it should, so we set cfp->sp at every call of this function. + // + // We use state.without_stack() to pass stack_size=0 to gen_save_sp() because we don't write + // VM stack slots on leaf calls, which leaves those stack slots uninitialized. ZJIT keeps + // live objects on the C stack, so they are protected from GC properly. + gen_prepare_call_with_gc(asm, &state.without_stack(), true); } /// Save the current SP on the CFP @@ -1247,6 +2779,7 @@ fn gen_save_sp(asm: &mut Assembler, stack_size: usize) { // code, and ZJIT's codegen currently assumes the SP register doesn't move, e.g. gen_param(). // So we don't update the SP register here. We could update the SP register to avoid using // an extra register for asm.lea(), but you'll need to manage the SP offset like YJIT does. + gen_incr_counter(asm, Counter::vm_write_sp_count); asm_comment!(asm, "save SP to CFP: {}", stack_size); let sp_addr = asm.lea(Opnd::mem(64, SP, stack_size as i32 * SIZEOF_VALUE_I32)); let cfp_sp = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP); @@ -1256,9 +2789,10 @@ fn gen_save_sp(asm: &mut Assembler, stack_size: usize) { /// Spill locals onto the stack. fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { // TODO: Avoid spilling locals that have been spilled before and not changed. + gen_incr_counter(asm, Counter::vm_write_locals_count); asm_comment!(asm, "spill locals"); for (idx, &insn_id) in state.locals().enumerate() { - asm.mov(Opnd::mem(64, SP, (-local_idx_to_ep_offset(jit.iseq, idx) - 1) * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + asm.mov(Opnd::mem(64, SP, (-local_idx_to_ep_offset(state.iseq, idx) - 1) * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); } } @@ -1266,6 +2800,7 @@ fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { // This function does not call gen_save_sp() at the moment because // gen_send_without_block_direct() spills stack slots above SP for arguments. + gen_incr_counter(asm, Counter::vm_write_stack_count); asm_comment!(asm, "spill stack"); for (idx, &insn_id) in state.stack().enumerate() { asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); @@ -1273,36 +2808,33 @@ fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { } /// Prepare for calling a C function that may call an arbitrary method. -/// Use gen_prepare_call_with_gc() if the method is leaf but allocates objects. +/// Use gen_prepare_leaf_call_with_gc() if the method is leaf but allocates objects. fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { // TODO: Lazily materialize caller frames when needed // Save PC for backtraces and allocation tracing - gen_save_pc(asm, state); + // and SP to avoid marking uninitialized stack slots + gen_prepare_call_with_gc(asm, state, false); - // Save SP and spill the virtual stack in case it raises an exception + // Spill the virtual stack in case it raises an exception // and the interpreter uses the stack for handling the exception - gen_save_sp(asm, state.stack().len()); gen_spill_stack(jit, asm, state); // Spill locals in case the method looks at caller Bindings gen_spill_locals(jit, asm, state); } -/// Prepare for calling a C function that may allocate objects and trigger GC. -/// Use gen_prepare_non_leaf_call() if it may also call an arbitrary method. -fn gen_prepare_call_with_gc(asm: &mut Assembler, state: &FrameState) { - // Save PC for allocation tracing - gen_save_pc(asm, state); - // Unlike YJIT, we don't need to save the stack to protect them from GC - // because the backend spills all live registers onto the C stack on asm.ccall(). -} - /// Frame metadata written by gen_push_frame() struct ControlFrame { recv: Opnd, - iseq: IseqPtr, + iseq: Option<IseqPtr>, cme: *const rb_callable_method_entry_t, frame_type: u32, + /// The [`VM_ENV_DATA_INDEX_SPECVAL`] slot of the frame. + /// For the type of frames we push, block handler or the parent EP. + specval: lir::Opnd, + /// Whether to write block_code = 0 at frame push time. + /// True when the callee ISEQ may write to block_code (has send/invokesuper/invokeblock). + write_block_code: bool, } /// Compile an interpreter frame @@ -1312,12 +2844,16 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C // See vm_push_frame() for details asm_comment!(asm, "push cme, specval, frame type"); // ep[-2]: cref of cme - let local_size = unsafe { get_iseq_body_local_table_size(frame.iseq) } as i32; + let local_size = if let Some(iseq) = frame.iseq { + (unsafe { get_iseq_body_local_table_size(iseq) }) as i32 + } else { + 0 + }; let ep_offset = state.stack().len() as i32 + local_size - argc as i32 + VM_ENV_DATA_SIZE as i32 - 1; + // ep[-2]: CME asm.store(Opnd::mem(64, SP, (ep_offset - 2) * SIZEOF_VALUE_I32), VALUE::from(frame.cme).into()); - // ep[-1]: block_handler or prev EP - // block_handler is not supported for now - asm.store(Opnd::mem(64, SP, (ep_offset - 1) * SIZEOF_VALUE_I32), VM_BLOCK_HANDLER_NONE.into()); + // ep[-1]: specval + asm.store(Opnd::mem(64, SP, (ep_offset - 1) * SIZEOF_VALUE_I32), frame.specval); // ep[0]: ENV_FLAGS asm.store(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32), frame.frame_type.into()); @@ -1327,31 +2863,57 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C } asm_comment!(asm, "push callee control frame"); - // cfp_opnd(RUBY_OFFSET_CFP_PC): written by the callee frame on side-exits or non-leaf calls - // cfp_opnd(RUBY_OFFSET_CFP_SP): written by the callee frame on side-exits or non-leaf calls - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), VALUE::from(frame.iseq).into()); + + if frame.iseq.is_some() { + // PC, SP, and ISEQ are written lazily by the callee on side-exits, non-leaf calls, or GC. + // cfp->jit_return will be written by gen_entry_point() on the callee after this frame push. + if frame.write_block_code { + asm_comment!(asm, "write block_code for iseq that may use it"); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + } + } else { + // C frames don't have a PC and ISEQ in normal operation. ISEQ frames set PC on gen_save_pc_for_gc(). + // When runtime checks are enabled we poison the PC for C frames so accidental reads stand out. + if let (None, Some(pc)) = (frame.iseq, PC_POISON) { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + } + let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); + // block_code must be written explicitly because the interpreter reads + // captured->code.ifunc directly from cfp->block_code (not through JITFrame). + // Without this, stale data from a previous frame occupying this CFP slot + // can be used as an ifunc pointer, causing a segfault. + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + // C frames share a single static JITFrame (rb_zjit_c_frame). Setting + // cfp->jit_return to the ZJIT_JIT_RETURN_C_FRAME sentinel tells + // CFP_ZJIT_FRAME() to use that shared frame, so we don't need to + // allocate a per-call JITFrame for C method pushes. + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), (ZJIT_JIT_RETURN_C_FRAME as usize).into()); + } + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); let ep = asm.lea(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); } -/// Return an operand we use for the basic block argument at a given index -fn param_opnd(idx: usize) -> Opnd { - // To simplify the implementation, allocate a fixed register or a stack slot for each basic block argument for now. - // Note that this is implemented here as opposed to automatically inside LIR machineries. - // TODO: Allow allocating arbitrary registers for basic block arguments - if idx < ALLOC_REGS.len() { - Opnd::Reg(ALLOC_REGS[idx]) - } else { - Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 1) as i32 * -SIZEOF_VALUE_I32) - } +/// Stack overflow check: fails if CFP<=SP at any point in the callee. +fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &FrameState, stack_growth: usize) { + asm_comment!(asm, "stack overflow check"); + // vm_push_frame() checks it against a decremented cfp, and CHECK_VM_STACK_OVERFLOW0 + // adds to the margin another control frame with `&bounds[1]`. + const { assert!(RUBY_SIZEOF_CONTROL_FRAME % SIZEOF_VALUE == 0, "sizeof(rb_control_frame_t) is a multiple of sizeof(VALUE)"); } + let cfp_growth = 2 * (RUBY_SIZEOF_CONTROL_FRAME / SIZEOF_VALUE); + let peak_offset = (cfp_growth + stack_growth) * SIZEOF_VALUE; + let stack_limit = asm.lea(Opnd::mem(64, SP, peak_offset as i32)); + asm.cmp(CFP, stack_limit); + asm.jbe(jit, side_exit(jit, state, StackOverflow)); } + /// Inverse of ep_offset_to_local_idx(). See ep_offset_to_local_idx() for details. -fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { +pub fn local_idx_to_ep_offset(iseq: IseqPtr, local_idx: usize) -> i32 { let local_size = unsafe { get_iseq_body_local_table_size(iseq) }; - local_size_and_idx_to_ep_offset(local_size as usize, local_idx) + local_size_and_idx_to_ep_offset(local_size.to_usize(), local_idx) } /// Convert the number of locals and a local index to an offset from the EP @@ -1366,34 +2928,54 @@ pub fn local_size_and_idx_to_bp_offset(local_size: usize, local_idx: usize) -> i } /// Convert ISEQ into High-level IR -fn compile_iseq(iseq: IseqPtr) -> Option<Function> { - let mut function = match iseq_to_hir(iseq) { +fn compile_iseq(iseq: IseqPtr) -> Result<Function, CompileError> { + // Convert ZJIT instructions back to bare instructions + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + + // Reject ISEQs with very large temp stacks. + // We cannot encode too large offsets to access locals in arm64. + let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) }; + if stack_max >= i8::MAX as u32 { + debug!("ISEQ stack too large: {stack_max}"); + return Err(CompileError::IseqStackTooLarge); + } + + let hir = trace_compile_phase("build_hir", || + crate::stats::with_time_stat(Counter::compile_hir_build_time_ns, || iseq_to_hir(iseq)) + ); + let mut function = match hir { Ok(function) => function, Err(err) => { - let name = crate::cruby::iseq_get_location(iseq, 0); - debug!("ZJIT: iseq_to_hir: {err:?}: {name}"); - return None; + debug!("ZJIT: iseq_to_hir: {err:?}: {}", iseq_get_location(iseq, 0)); + return Err(CompileError::ParseError(err)); } }; if !get_option!(disable_hir_opt) { - function.optimize(); + trace_compile_phase("optimize", || function.optimize()); } function.dump_hir(); - #[cfg(debug_assertions)] - if let Err(err) = function.validate() { - debug!("ZJIT: compile_iseq: {err:?}"); - return None; - } - Some(function) + Ok(function) } -/// Build a Target::SideExit for non-PatchPoint instructions -fn side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReason) -> Target { - build_side_exit(jit, state, reason, None) +/// Build a Target::SideExit +fn side_exit(jit: &JITState, state: &FrameState, reason: SideExitReason) -> Target { + let exit = build_side_exit(jit, state); + Target::SideExit { exit, reason } } -/// Build a Target::SideExit out of a FrameState -fn build_side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReason, label: Option<Label>) -> Target { +/// Build a Target::SideExit that optionally triggers exit_recompile on the exit path. +fn side_exit_with_recompile(jit: &JITState, state: &FrameState, reason: SideExitReason, recompile: Option<Recompile>) -> Target { + let mut exit = build_side_exit(jit, state); + exit.recompile = recompile.map(|strategy| SideExitRecompile { + iseq: Opnd::Value(VALUE::from(state.iseq)), + insn_idx: state.insn_idx() as u32, + strategy, + }); + Target::SideExit { exit, reason } +} + +/// Build a side-exit context +fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { let mut stack = Vec::new(); for &insn_id in state.stack() { stack.push(jit.get_opnd(insn_id)); @@ -1404,109 +2986,238 @@ fn build_side_exit(jit: &mut JITState, state: &FrameState, reason: SideExitReaso locals.push(jit.get_opnd(insn_id)); } - let target = Target::SideExit { - pc: state.pc, + SideExit{ + pc: Opnd::const_ptr(state.pc), stack, locals, - reason, - label, - }; - target -} - -/// Return true if a given ISEQ is known to escape EP to the heap on entry. -/// -/// As of vm_push_frame(), EP is always equal to BP. However, after pushing -/// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP. -fn iseq_entry_escapes_ep(iseq: IseqPtr) -> bool { - match unsafe { get_iseq_body_type(iseq) } { - // <main> frame is always associated to TOPLEVEL_BINDING. - ISEQ_TYPE_MAIN | - // Kernel#eval uses a heap EP when a Binding argument is not nil. - ISEQ_TYPE_EVAL => true, - _ => false, + iseq: state.iseq, + recompile: None, } } -/// Returne the maximum number of arguments for a block in a given function -fn max_num_params(function: &Function) -> usize { - let reverse_post_order = function.rpo(); - reverse_post_order.iter().map(|&block_id| { - let block = function.block(block_id); - block.params().len() - }).max().unwrap_or(0) -} - #[cfg(target_arch = "x86_64")] macro_rules! c_callable { ($(#[$outer:meta])* - fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $vis:vis fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { $(#[$outer])* - extern "sysv64" fn $f $args $(-> $ret)? $body + $vis extern "sysv64" fn $f $args $(-> $ret)? $body }; } #[cfg(target_arch = "aarch64")] macro_rules! c_callable { ($(#[$outer:meta])* - fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { + $vis:vis fn $f:ident $args:tt $(-> $ret:ty)? $body:block) => { $(#[$outer])* - extern "C" fn $f $args $(-> $ret)? $body + $vis extern "C" fn $f $args $(-> $ret)? $body }; } +#[cfg(test)] pub(crate) use c_callable; c_callable! { + /// Called from JIT side-exit code to profile operands and trigger recompilation. + /// For send instructions (argc >= 0): profiles receiver + args from the stack. + /// For shape guard exits (argc == -1): profiles self from the CFP. + /// Once enough profiles are gathered, invalidates the ISEQ for recompilation. + pub(crate) fn exit_recompile(ec: EcPtr, iseq_raw: VALUE, insn_idx: u32, argc: i32) { + // Fast check before taking the VM lock: skip if already invalidated or + // at the version limit. This avoids expensive lock acquisition on every + // shape guard exit after the recompile has already been triggered. + { + let iseq: IseqPtr = iseq_raw.as_iseq(); + let payload = get_or_create_iseq_payload(iseq); + let already_done = payload.versions.last() + .map_or(false, |v| unsafe { v.as_ref() }.is_invalidated()) + || payload.versions.len() >= max_iseq_versions(); + if already_done { + return; + } + } + + with_vm_lock(src_loc!(), || { + let iseq: IseqPtr = iseq_raw.as_iseq(); + let payload = get_or_create_iseq_payload(iseq); + + // For no-profile sends, skip if already profiled at this insn_idx. + // For shape guard exits (argc == -1), always re-profile because the + // original YARV profiles were monomorphic but runtime showed new shapes. + if argc >= 0 && payload.profile.done_profiling_at(insn_idx as usize) { + return; + } + + with_time_stat(Counter::profile_time_ns, || { + let cfp = unsafe { get_ec_cfp(ec) }; + + let should_recompile = if argc >= 0 { + let sp = unsafe { get_cfp_sp(cfp) }; + // Profile the receiver and arguments for this send instruction + payload.profile.profile_send_at(iseq, insn_idx as usize, sp, argc as usize) + } else { + // Profile self for shape guard exits (argc == -1) + let self_val = unsafe { get_cfp_self(cfp) }; + payload.profile.profile_self_at(iseq, insn_idx as usize, self_val) + }; + + // Once we have enough profiles, invalidate and recompile the ISEQ + if should_recompile { + if let Some(version) = payload.versions.last_mut() { + let cb = ZJITState::get_code_block(); + invalidate_iseq_version(cb, iseq, version); + cb.mark_all_executable(); + } + } + }); + }); + } +} + +c_callable! { /// Generated code calls this function with the SysV calling convention. See [gen_function_stub]. /// This function is expected to be called repeatedly when ZJIT fails to compile the stub. /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. - fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE) -> *const u8 { - with_vm_lock(src_loc!(), || { - // gen_push_frame() doesn't set PC, so we need to set them before exit. - // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. - let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const RefCell<IseqCall>) }; - let iseq = iseq_call.borrow().iseq; - let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; // TODO: handle opt_pc once supported + fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE, ec: EcPtr) -> *const u8 { + // Make sure cfp is ready to be scanned by other Ractors and GC before taking the barrier + { + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); + let params = unsafe { iseq.params() }; + let entry_idx = iseq_call.jit_entry_idx.to_usize(); + let entry_insn_idx = params.opt_table_slice().get(entry_idx) + .unwrap_or_else(|| panic!("function_stub: opt_table out of bounds. {params:#?}, entry_idx={entry_idx}")) + .as_u32(); + // gen_push_frame() doesn't set PC or ISEQ, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC and ISEQ. + // Clear jit_return so the interpreter reads cfp->pc and cfp->iseq directly. + let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idx) }; unsafe { rb_set_cfp_pc(cfp, pc) }; + unsafe { (*cfp)._iseq = iseq }; + unsafe { (*cfp).jit_return = std::ptr::null_mut() }; + } - // JIT-to-JIT calls don't set SP or fill nils to uninitialized (non-argument) locals. - // We need to set them if we side-exit from function_stub_hit. - fn spill_stack(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE) { + with_vm_lock(src_loc!(), || { + // Re-create the Rc inside the VM lock because IseqCall's interior + // mutability (Cell<IseqPtr>) requires exclusive access. + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); + let argc = iseq_call.argc; + let num_opts_filled = iseq_call.jit_entry_idx; + + // JIT-to-JIT calls don't eagerly fill nils to non-parameter locals. + // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. + fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, argc: u16, num_opts_filled: u16, compile_error: &CompileError) { unsafe { + // Caller frames are materialized by the materialize_exit trampoline before unwinding native frames. + // The current frame's pc and iseq are already set by function_stub_hit before this point. + // Set SP which gen_push_frame() doesn't set rb_set_cfp_sp(cfp, sp); - // Fill nils to uninitialized (non-argument) locals - let local_size = get_iseq_body_local_table_size(iseq) as usize; - let num_params = get_iseq_body_param_size(iseq) as usize; - let base = sp.offset(-local_size_and_idx_to_bp_offset(local_size, num_params) as isize); - slice::from_raw_parts_mut(base, local_size - num_params).fill(Qnil); + let local_size = get_iseq_body_local_table_size(iseq).to_usize(); + let params = iseq.params(); + let params_size = params.size.to_usize(); + let frame_base = sp.offset(-local_size_and_idx_to_bp_offset(local_size, 0) as isize); + let locals = slice::from_raw_parts_mut(frame_base, local_size); + // Fill nils to uninitialized (non-parameter) locals + locals.get_mut(params_size..).unwrap_or_default().fill(Qnil); + + // SendDirect packs args without gaps for unfilled optionals. + // When we exit to the interpreter, we need to shift args right + // to create the gap and nil-fill the unfilled optional slots. + // + // Example: def target(req, a = a, b = b, kw:); target(1, kw: 2) + // lead_num=1, opt_num=2, opts_filled=0, argc=2 + // + // locals[] as placed by SendDirect (argc=2, no gaps): + // [req, kw_val, ?, ?, ?, ...] + // 0 1 + // ^----caller's args----^ + // + // locals[] expected by interpreter (params_size=4): + // [req, a, b, kw_val, ?, ...] + // 0 1 2 3 + // ^nil ^nil^--moved--^ + // + // gap_start = lead_num + opts_filled = 1 + // gap_end = lead_num + opt_num = 3 + // We move locals[gap_start..argc] to locals[gap_end..], then + // nil-fill locals[gap_start..gap_end]. + let opt_num: usize = params.opt_num.try_into().expect("ISEQ opt_num should be non-negative"); + let opts_filled = num_opts_filled.to_usize(); + let opts_unfilled = opt_num.saturating_sub(opts_filled); + if opts_unfilled > 0 { + let argc = argc.to_usize(); + let lead_num: usize = params.lead_num.try_into().expect("ISEQ lead_num should be non-negative"); + let param_locals = &mut locals[..params_size]; + // Gap of unspecified optional parameters + let gap_start = lead_num + opts_filled; + let gap_end = lead_num + opt_num; + // When there are arguments in the gap, shift them past the gap + let args_overlapping_gap = gap_start..argc; + if !args_overlapping_gap.is_empty() { + assert!( + gap_end.checked_add(args_overlapping_gap.len()) + .is_some_and(|new_end| new_end <= param_locals.len()) , + "shift past gap out-of-bounds. params={params:#?} args_overlapping_gap={args_overlapping_gap:?}" + ); + param_locals.copy_within(args_overlapping_gap, gap_end); + } + // Nil-fill the now-vacant optional parameter slots + param_locals[gap_start..gap_end].fill(Qnil); + } + } + + // Increment a compile error counter for --zjit-stats + if get_option!(stats) { + incr_counter_by(exit_counter_for_compile_error(compile_error), 1); } } - // If we already know we can't compile the ISEQ, fail early without cb.mark_all_executable(). + // If we already know we can't compile the ISEQ, or there is insufficient native + // stack space, fail early without cb.mark_all_executable(). // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. let cb = ZJITState::get_code_block(); + let native_stack_full = unsafe { rb_ec_stack_check(ec as _) } != 0; let payload = get_or_create_iseq_payload(iseq); - if cb.has_dropped_bytes() || payload.status == IseqStatus::CantCompile { + // cfp is the callee's (this ISEQ's) frame here, so its method entry gives + // the owning class and thus whether `self` is always a heap object. + let cme = unsafe { rb_vm_frame_method_entry(cfp) }; + payload.self_is_heap_object = !cme.is_null() + && iseq_self_is_heap_object(iseq, unsafe { (*cme).owner }); + let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); + let compile_error = match last_status { + Some(IseqStatus::CantCompile(err)) => Some(err), + _ if cb.has_dropped_bytes() => Some(&CompileError::OutOfMemory), + _ if native_stack_full => { + incr_counter!(skipped_native_stack_full); + Some(&CompileError::NativeStackTooLarge) + }, + _ => None, + }; + if let Some(compile_error) = compile_error { // We'll use this Rc again, so increment the ref count decremented by from_raw. - unsafe { Rc::increment_strong_count(iseq_call_ptr as *const RefCell<IseqCall>); } + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } - // Exit to the interpreter - spill_stack(iseq, cfp, sp); - return ZJITState::get_exit_trampoline().raw_ptr(cb); + prepare_for_exit(iseq, cfp, sp, argc, num_opts_filled, compile_error); + return ZJITState::get_materialize_exit_trampoline_with_counter().raw_ptr(cb); } // Otherwise, attempt to compile the ISEQ. We have to mark_all_executable() beyond this point. let code_ptr = with_time_stat(compile_time_ns, || function_stub_hit_body(cb, &iseq_call)); - let code_ptr = if let Some(code_ptr) = code_ptr { - code_ptr - } else { - // Exit to the interpreter - spill_stack(iseq, cfp, sp); - ZJITState::get_exit_trampoline() - }; + if code_ptr.is_ok() { + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.incoming.push(iseq_call); + } + } + let code_ptr = code_ptr.unwrap_or_else(|compile_error| { + // We'll use this Rc again, so increment the ref count decremented by from_raw. + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } + + prepare_for_exit(iseq, cfp, sp, argc, num_opts_filled, &compile_error); + ZJITState::get_materialize_exit_trampoline_with_counter() + }); cb.mark_all_executable(); code_ptr.raw_ptr(cb) }) @@ -1514,37 +3225,36 @@ c_callable! { } /// Compile an ISEQ for a function stub -fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &Rc<RefCell<IseqCall>>) -> Option<CodePtr> { +fn function_stub_hit_body(cb: &mut CodeBlock, iseq_call: &IseqCallRef) -> Result<CodePtr, CompileError> { // Compile the stubbed ISEQ - let Some((code_ptr, iseq_calls)) = gen_iseq(cb, iseq_call.borrow().iseq) else { - debug!("Failed to compile iseq: gen_iseq failed: {}", iseq_get_location(iseq_call.borrow().iseq, 0)); - return None; - }; - - // Stub callee ISEQs for JIT-to-JIT calls - for callee_iseq_call in iseq_calls.iter() { - gen_iseq_call(cb, iseq_call.borrow().iseq, callee_iseq_call)?; - } + let IseqCodePtrs { jit_entry_ptrs, .. } = gen_iseq(cb, iseq_call.iseq.get(), None).inspect_err(|err| { + debug!("{err:?}: gen_iseq failed: {}", iseq_get_location(iseq_call.iseq.get(), 0)); + })?; // Update the stub to call the code pointer - let code_addr = code_ptr.raw_ptr(cb); - let iseq = iseq_call.borrow().iseq; - iseq_call.borrow_mut().regenerate(cb, |asm| { - asm_comment!(asm, "call compiled function: {}", iseq_get_location(iseq, 0)); - asm.ccall(code_addr, vec![]); + let jit_entry_ptr = jit_entry_ptrs[iseq_call.jit_entry_idx.to_usize()]; + let code_addr = jit_entry_ptr.raw_ptr(cb); + let iseq = iseq_call.iseq.get(); + trace_compile_phase("compile_stub", || { + iseq_call.regenerate(cb, |asm| { + asm_comment!(asm, "call compiled function: {}", iseq_get_location(iseq, 0)); + asm.ccall_into(C_RET_OPND, code_addr, vec![]); + }); }); - Some(code_ptr) + Ok(jit_entry_ptr) } -/// Compile a stub for an ISEQ called by SendWithoutBlockDirect -fn gen_function_stub(cb: &mut CodeBlock, iseq_call: Rc<RefCell<IseqCall>>) -> Option<CodePtr> { - let mut asm = Assembler::new(); - asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.borrow().iseq, 0)); +/// Compile a stub for an ISEQ called by SendDirect +fn gen_function_stub(cb: &mut CodeBlock, iseq_call: IseqCallRef) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id("gen_function_stub"); + asm_comment!(asm, "Stub: {}", iseq_get_location(iseq_call.iseq.get(), 0)); // Call function_stub_hit using the shared trampoline. See `gen_function_stub_hit_trampoline`. // Use load_into instead of mov, which is split on arm64, to avoid clobbering ALLOC_REGS. - asm.load_into(SCRATCH_OPND, Opnd::const_ptr(Rc::into_raw(iseq_call).into())); + asm.load_into(scratch_reg, Opnd::const_ptr(Rc::into_raw(iseq_call))); + asm.cpush(scratch_reg); asm.jmp(ZJITState::get_function_stub_hit_trampoline().into()); asm.compile(cb).map(|(code_ptr, gc_offsets)| { @@ -1553,34 +3263,68 @@ fn gen_function_stub(cb: &mut CodeBlock, iseq_call: Rc<RefCell<IseqCall>>) -> Op }) } -/// Generate a trampoline that is used when a -pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Option<CodePtr> { - let mut asm = Assembler::new(); +/// Generate a trampoline that is used when a function stub is called. +/// See [gen_function_stub] for how it's used. +pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { + let (mut asm, scratch_reg) = Assembler::new_with_scratch_reg(); + asm.new_block_without_id("function_stub_hit_trampoline"); asm_comment!(asm, "function_stub_hit trampoline"); + asm.cpop_into(scratch_reg); + // Maintain alignment for x86_64, and set up a frame for arm64 properly - asm.frame_setup(&[], 0); + asm.frame_setup(&[]); asm_comment!(asm, "preserve argument registers"); - for ® in ALLOC_REGS.iter() { - asm.cpush(Opnd::Reg(reg)); + + for pair in ALLOC_REGS.chunks(2) { + match *pair { + [reg0, reg1] => { + asm.cpush_pair(Opnd::Reg(reg0), Opnd::Reg(reg1)); + } + [reg] => { + asm.cpush(Opnd::Reg(reg)); + } + _ => unreachable!("chunks(2)") + } + } + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpush(Opnd::Reg(ALLOC_REGS[0])); // maintain alignment for x86_64 } - const { assert!(ALLOC_REGS.len() % 2 == 0, "x86_64 would need to push one more if we push an odd number of regs"); } + // We can't directly pass the scratch register in to the ccall because + // we're going to have parallel move automatically handle coping registers + // in to the C calling convention and the parallel move algorithm needs + // a scratch register to break any cycles. If we use the scratch register + // as a C call parameter, then parallel move wouldn't be able to break + // cycles without clobbering something + asm.mov(C_ARG_OPNDS[0], scratch_reg); // Compile the stubbed ISEQ - let jump_addr = asm_ccall!(asm, function_stub_hit, SCRATCH_OPND, CFP, SP); - asm.mov(SCRATCH_OPND, jump_addr); + let jump_addr = asm_ccall!(asm, function_stub_hit, C_ARG_OPNDS[0], CFP, SP, EC); + asm.mov(scratch_reg, jump_addr); asm_comment!(asm, "restore argument registers"); - for ® in ALLOC_REGS.iter().rev() { - asm.cpop_into(Opnd::Reg(reg)); + if cfg!(target_arch = "x86_64") && ALLOC_REGS.len() % 2 == 1 { + asm.cpop_into(Opnd::Reg(ALLOC_REGS[0])); + } + + for pair in ALLOC_REGS.chunks(2).rev() { + match *pair { + [reg] => { + asm.cpop_into(Opnd::Reg(reg)); + } + [reg0, reg1] => { + asm.cpop_pair_into(Opnd::Reg(reg1), Opnd::Reg(reg0)); + } + _ => unreachable!("chunks(2)") + } } // Discard the current frame since the JIT function will set it up again asm.frame_teardown(&[]); - // Jump to SCRATCH_OPND so that cpop_into() doesn't clobber it - asm.jmp_opnd(SCRATCH_OPND); + // Jump to scratch_reg so that cpop_into() doesn't clobber it + asm.jmp_opnd(scratch_reg); asm.compile(cb).map(|(code_ptr, gc_offsets)| { assert_eq!(gc_offsets.len(), 0); @@ -1589,11 +3333,12 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Option<CodePtr> { } /// Generate a trampoline that is used when a function exits without restoring PC and the stack -pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Option<CodePtr> { +pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError> { let mut asm = Assembler::new(); + asm.new_block_without_id("exit_trampoline"); asm_comment!(asm, "side-exit trampoline"); - asm.frame_teardown(&[]); // matching the setup in :bb0-prologue: + asm.frame_teardown(&[]); // matching the setup in gen_entry_point() asm.cret(Qundef.into()); asm.compile(cb).map(|(code_ptr, gc_offsets)| { @@ -1602,60 +3347,131 @@ pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Option<CodePtr> { }) } -fn gen_push_opnds(jit: &mut JITState, asm: &mut Assembler, opnds: &[Opnd]) -> lir::Opnd { - let n = opnds.len(); +/// Generate a trampoline that materializes ZJIT frames before unwinding native frames. +pub fn gen_materialize_exit_trampoline(cb: &mut CodeBlock, exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> { + unsafe extern "C" { + fn rb_zjit_materialize_frames(ec: EcPtr, cfp: CfpPtr); + } - // Calculate the compile-time NATIVE_STACK_PTR offset from NATIVE_BASE_PTR - // At this point, frame_setup(&[], jit.c_stack_slots) has been called, - // which allocated aligned_stack_bytes(jit.c_stack_slots) on the stack - let frame_size = aligned_stack_bytes(jit.c_stack_slots); - let allocation_size = aligned_stack_bytes(n); + let mut asm = Assembler::new(); + asm.new_block_without_id("materialize_exit_trampoline"); - asm_comment!(asm, "allocate {} bytes on C stack for {} values", allocation_size, n); - asm.sub_into(NATIVE_STACK_PTR, allocation_size.into()); + asm_comment!(asm, "materialize ZJIT frames"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + asm_ccall!(asm, rb_zjit_materialize_frames, EC, CFP); + asm.jmp(Target::CodePtr(exit_trampoline)); - // Calculate the total offset from NATIVE_BASE_PTR to our buffer - let total_offset_from_base = (frame_size + allocation_size) as i32; + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) +} - for (idx, &opnd) in opnds.iter().enumerate() { - let slot_offset = -total_offset_from_base + (idx as i32 * SIZEOF_VALUE_I32); - asm.mov( - Opnd::mem(VALUE_BITS, NATIVE_BASE_PTR, slot_offset), - opnd - ); - } +/// Generate a trampoline that increments exit_compilation_failure and jumps to materialize_exit_trampoline. +pub fn gen_materialize_exit_trampoline_with_counter(cb: &mut CodeBlock, materialize_exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + asm.new_block_without_id("materialize_exit_trampoline_with_counter"); + + asm_comment!(asm, "function stub exit trampoline"); + gen_incr_counter(&mut asm, exit_compile_error); + asm.jmp(Target::CodePtr(materialize_exit_trampoline)); - asm.lea(Opnd::mem(64, NATIVE_BASE_PTR, -total_offset_from_base)) + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(gc_offsets.len(), 0); + code_ptr + }) } -fn gen_pop_opnds(asm: &mut Assembler, opnds: &[Opnd]) { - asm_comment!(asm, "restore C stack pointer"); - let allocation_size = aligned_stack_bytes(opnds.len()); - asm.add_into(NATIVE_STACK_PTR, allocation_size.into()); +/// Reserve native stack space and write operands into it. +fn gen_push_opnds(jit: &JITState, asm: &mut Assembler, opnds: &[Opnd]) -> lir::Opnd { + let argv = if opnds.len() > 0 { + // Make sure the Assembler will reserve a sufficient stack size for given opnds + asm_comment!(asm, "allocate space on C stack for {} values", opnds.len()); + asm.alloc_stack(jit, opnds.len()) + } else { + asm_comment!(asm, "no opnds to allocate"); + Opnd::UImm(0) + }; + + // Write operands into stack slots allocated by asm.alloc_stack() + for (idx, &opnd) in opnds.iter().enumerate() { + asm.mov(Opnd::mem(VALUE_BITS, argv, idx as i32 * SIZEOF_VALUE_I32), opnd); + } + + argv } fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, opt: usize, values: Vec<Opnd>, state: &FrameState) -> Opnd { gen_prepare_non_leaf_call(jit, asm, state); let first_opnd_ptr = gen_push_opnds(jit, asm, &values); - - let tmp_ary = asm_ccall!(asm, rb_ary_tmp_new_from_values, Opnd::Imm(0), values.len().into(), first_opnd_ptr); - let result = asm_ccall!(asm, rb_reg_new_ary, tmp_ary, opt.into()); - asm_ccall!(asm, rb_ary_clear, tmp_ary); - - gen_pop_opnds(asm, &values); - - result + asm_ccall!(asm, rb_reg_new_from_values, values.len().into(), first_opnd_ptr, opt.into()) } fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec<Opnd>, state: &FrameState) -> Opnd { gen_prepare_non_leaf_call(jit, asm, state); let first_string_ptr = gen_push_opnds(jit, asm, &strings); - let result = asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr); - gen_pop_opnds(asm, &strings); + asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr) +} + +// Generate RSTRING_PTR +fn get_string_ptr(asm: &mut Assembler, string: Opnd) -> Opnd { + asm_comment!(asm, "get string pointer for embedded or heap"); + let string = asm.load_mem(string); + let flags = Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RBASIC_FLAGS); + asm.test(flags, (RSTRING_NOEMBED as u64).into()); + let heap_ptr = asm.load(Opnd::mem( + usize::BITS as u8, + string, + RUBY_OFFSET_RSTRING_AS_HEAP_PTR, + )); + // Load the address of the embedded array + // (struct RString *)(obj)->as.ary + let ary = asm.lea(Opnd::mem(VALUE_BITS, string, RUBY_OFFSET_RSTRING_AS_ARY)); + asm.csel_nz(heap_ptr, ary) +} + +fn gen_string_getbyte(asm: &mut Assembler, string: Opnd, index: Opnd) -> Opnd { + let string_ptr = get_string_ptr(asm, string); + // TODO(max): Use SIB indexing here once the backend supports it + let string_ptr = asm.add(string_ptr, index); + let byte = asm.load(Opnd::mem(8, string_ptr, 0)); + // Zero-extend the byte to 64 bits + let byte = byte.with_num_bits(64); + let byte = asm.and(byte, 0xFF.into()); + // Tag the byte + let byte = asm.lshift(byte, Opnd::UImm(1)); + asm.or(byte, Opnd::UImm(1)) +} + +fn gen_string_setbyte_fixnum(asm: &mut Assembler, string: Opnd, index: Opnd, value: Opnd) -> Opnd { + // rb_str_setbyte is not leaf, but we guard types and index ranges in HIR + asm_ccall!(asm, rb_str_setbyte, string, index, value) +} + +fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_str_buf_append, string, val) +} - result +fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, state); + asm_ccall!(asm, rb_jit_str_concat_codepoint, string, val) +} + +/// Generate a JIT entry that just increments exit_compilation_failure and exits +fn gen_compile_error_counter(cb: &mut CodeBlock, compile_error: &CompileError) -> Result<CodePtr, CompileError> { + let mut asm = Assembler::new(); + asm.new_block_without_id("compile_error_counter"); + gen_incr_counter(&mut asm, exit_compile_error); + gen_incr_counter(&mut asm, exit_counter_for_compile_error(compile_error)); + asm.cret(Qundef.into()); + + asm.compile(cb).map(|(code_ptr, gc_offsets)| { + assert_eq!(0, gc_offsets.len()); + code_ptr + }) } /// Given the number of spill slots needed for a function, return the number of bytes @@ -1668,8 +3484,53 @@ fn aligned_stack_bytes(num_slots: usize) -> usize { } impl Assembler { + /// Allocate stack space on top of the stack slots reserved for JITFrame, + /// and return a pointer to the allocated space. + fn alloc_stack(&mut self, jit: &JITState, stack_size: usize) -> Opnd { + let total_stack_size = jit.jit_frame_size + stack_size; + self.stack_base_idx = self.stack_base_idx.max(total_stack_size); + // high addr + // +------------------------+ + // | return address | + // +------------------------+ + // | previous frame pointer | <- NATIVE_BASE_PTR == cfp->jit_return + // +------------------------+ + // | JITFrame pointer | <- jit.jit_frame_size, read by CFP_ZJIT_FRAME(cfp) + // +------------------------+ + // | opnds.last() | + // +------------------------+ + // | ... | + // +------------------------+ + // | opnds.first() | <- pointer returned by alloc_stack() + // +------------------------+ + // | register spill slots | if any + // +------------------------+ + // | FrameSetup align slot | if needed + // +------------------------+ + // low addr + self.sub(NATIVE_BASE_PTR, (SIZEOF_VALUE * total_stack_size).into()) + } + + /// Emits a load for memory based operands and returns a vreg, + /// otherwise returns recv. + fn load_mem(&mut self, recv: Opnd) -> Opnd { + match recv { + Opnd::VReg { .. } | Opnd::Reg(_) => recv, + _ => self.load(recv), + } + } + + /// Emits a load for constant based operands and returns a vreg, + /// otherwise returns recv. + fn load_imm(&mut self, recv: Opnd) -> Opnd { + match recv { + Opnd::Value { .. } | Opnd::UImm(_) | Opnd::Imm(_) => self.load(recv), + _ => recv, + } + } + /// Make a C call while marking the start and end positions for IseqCall - fn ccall_with_iseq_call(&mut self, fptr: *const u8, opnds: Vec<Opnd>, iseq_call: &Rc<RefCell<IseqCall>>) -> Opnd { + fn ccall_with_iseq_call(&mut self, fptr: *const u8, opnds: Vec<Opnd>, iseq_call: &IseqCallRef) -> Opnd { // We need to create our own branch rc objects so that we can move the closure below let start_iseq_call = iseq_call.clone(); let end_iseq_call = iseq_call.clone(); @@ -1678,20 +3539,45 @@ impl Assembler { fptr, opnds, move |code_ptr, _| { - start_iseq_call.borrow_mut().start_addr.set(Some(code_ptr)); + start_iseq_call.start_addr.set(Some(code_ptr)); }, move |code_ptr, _| { - end_iseq_call.borrow_mut().end_addr.set(Some(code_ptr)); + end_iseq_call.end_addr.set(Some(code_ptr)); }, ) } } +/// Store info about a JIT entry point +pub struct JITEntry { + /// Index that corresponds to an entry in [crate::cruby::IseqParameters::opt_table_slice] + jit_entry_idx: usize, + /// Position where the entry point starts + start_addr: Cell<Option<CodePtr>>, +} + +impl JITEntry { + /// Allocate a new JITEntry + fn new(jit_entry_idx: usize) -> Rc<RefCell<Self>> { + let jit_entry = JITEntry { + jit_entry_idx, + start_addr: Cell::new(None), + }; + Rc::new(RefCell::new(jit_entry)) + } +} + /// Store info about a JIT-to-JIT call #[derive(Debug)] pub struct IseqCall { /// Callee ISEQ that start_addr jumps to - pub iseq: IseqPtr, + pub iseq: Cell<IseqPtr>, + + /// Index that corresponds to an entry in [crate::cruby::IseqParameters::opt_table_slice] + jit_entry_idx: u16, + + /// Argument count passing to the HIR function + argc: u16, /// Position where the call instruction starts start_addr: Cell<Option<CodePtr>>, @@ -1700,24 +3586,61 @@ pub struct IseqCall { end_addr: Cell<Option<CodePtr>>, } +pub type IseqCallRef = Rc<IseqCall>; + impl IseqCall { /// Allocate a new IseqCall - fn new(iseq: IseqPtr) -> Rc<RefCell<Self>> { + fn new(iseq: IseqPtr, jit_entry_idx: u16, argc: u16) -> IseqCallRef { let iseq_call = IseqCall { - iseq, + iseq: Cell::new(iseq), start_addr: Cell::new(None), end_addr: Cell::new(None), + jit_entry_idx, + argc, }; - Rc::new(RefCell::new(iseq_call)) + Rc::new(iseq_call) } /// Regenerate a IseqCall with a given callback fn regenerate(&self, cb: &mut CodeBlock, callback: impl Fn(&mut Assembler)) { - cb.with_write_ptr(self.start_addr.get().unwrap(), |cb| { + cb.with_write_ptr(self.start_addr.get().expect("expected a start address"), |cb| { let mut asm = Assembler::new(); + asm.new_block_without_id("regenerate"); callback(&mut asm); asm.compile(cb).unwrap(); assert_eq!(self.end_addr.get().unwrap(), cb.get_write_ptr()); }); } } + +type PerfSymbol = Rc<RefCell<Option<(CodePtr, String)>>>; + +/// Mark the start of a perf symbol range via pos_marker. +/// Returns a handle to pass to perf_symbol_range_end. +pub fn perf_symbol_range_start(asm: &mut Assembler, symbol_name: &str) -> PerfSymbol { + let symbol_name = symbol_name.to_string(); + let perf_symbol: PerfSymbol = Rc::new(RefCell::new(None)); + let current = perf_symbol.clone(); + asm.pos_marker(move |start, _| { + let mut current = current.borrow_mut(); + assert!(current.is_none(), "perf symbol range already open"); + *current = Some((start, symbol_name.clone())); + }); + perf_symbol +} + +/// Mark the end of a perf symbol range via pos_marker. +pub fn perf_symbol_range_end(asm: &mut Assembler, perf_symbol: &PerfSymbol) { + let current = perf_symbol.clone(); + asm.pos_marker(move |end, cb| { + if let Some((start, name)) = current.borrow_mut().take() { + let start_addr = start.raw_addr(cb); + let code_size = end.raw_addr(cb) - start_addr; + register_with_perf(name, start_addr, code_size); + } + }); +} + +#[cfg(test)] +#[path = "codegen_tests.rs"] +mod tests; diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs new file mode 100644 index 0000000000..9b76690d5b --- /dev/null +++ b/zjit/src/codegen_tests.rs @@ -0,0 +1,5814 @@ +#![cfg(test)] + +use super::{gen_insn, JITState}; +use crate::asm::CodeBlock; +use crate::backend::lir::Assembler; +use crate::codegen::max_iseq_versions; +use crate::cruby::*; +use crate::hir::{Insn, iseq_to_hir}; +use crate::options::{rb_zjit_prepare_options, set_call_threshold}; +use crate::payload::IseqVersion; +use crate::hir::tests::hir_build_tests::assert_contains_opcode; +use crate::payload::*; +use insta::assert_snapshot; + +#[test] +fn test_breakpoint_hir_codegen() { + rb_zjit_prepare_options(); + + eval("def test_breakpoint_hir_codegen = nil"); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", "test_breakpoint_hir_codegen")); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + let breakpoint = function.push_insn(function.entries_block, Insn::BreakPoint); + + let mut jit = JITState::new( + IseqVersion::new(iseq), + function.num_insns(), + function.num_blocks(), + 0, + ); + let mut asm = Assembler::new(); + asm.new_block_without_id("test"); + let mut cb = CodeBlock::new_dummy(); + + gen_insn(&mut cb, &mut jit, &mut asm, &function, breakpoint, &function.find(breakpoint)).unwrap(); + asm.compile_with_num_regs(&mut cb, 0); + + #[cfg(target_arch = "x86_64")] + assert_eq!(cb.hexdump(), "cc"); + + #[cfg(target_arch = "aarch64")] + assert_eq!(cb.hexdump(), "00003ed4"); +} + +#[test] +fn test_call_itself() { + assert_snapshot!(inspect(" + def test = 42.itself + test + test + "), @"42"); +} + +#[test] +fn test_nil() { + assert_snapshot!(inspect(" + def test = nil + test + test + "), @"nil"); +} + +#[test] +fn test_putobject() { + assert_snapshot!(inspect(" + def test = 1 + test + test + "), @"1"); +} + +#[test] +fn test_dupstring() { + eval(r##" + def test = "#{""}" + test + "##); + assert_contains_opcode("test", YARVINSN_dupstring); + assert_snapshot!(assert_compiles(r##"test"##), @r#""""#); +} + +#[test] +fn test_dupchilledstring() { + eval(r#" + def test = "" + test + "#); + assert_contains_opcode("test", YARVINSN_dupchilledstring); + assert_snapshot!(assert_compiles(r#"test"#), @r#""""#); +} + +#[test] +fn test_leave_param() { + assert_snapshot!(inspect(" + def test(n) = n + test(5) + test(5) + "), @"5"); +} + +#[test] +fn test_getglobal_with_warning() { + eval(r#" + Warning[:deprecated] = true + + module Warning + def warn(message) + raise + end + end + + def test + $= + rescue + "rescued" + end + $VERBOSE = true + test + "#); + assert_contains_opcode("test", YARVINSN_getglobal); + assert_snapshot!(assert_compiles(r#"test"#), @r#""rescued""#); +} + +#[test] +fn test_setglobal() { + eval(" + def test + $a = 1 + $a + end + test + "); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(assert_compiles("test"), @"1"); +} + +#[test] +fn test_string_intern() { + eval(r#" + def test + :"foo#{123}" + end + test + "#); + assert_contains_opcode("test", YARVINSN_intern); + assert_snapshot!(assert_compiles(r#"test"#), @":foo123"); +} + +#[test] +fn test_duphash() { + eval(" + def test + {a: 1} + end + test + "); + assert_contains_opcode("test", YARVINSN_duphash); + assert_snapshot!(assert_compiles("test"), @"{a: 1}"); +} + +#[test] +fn test_pushtoarray() { + eval(" + def test + [*[], 1, 2, 3] + end + test + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(assert_compiles("test"), @"[1, 2, 3]"); +} + +#[test] +fn test_splatarray_new_array() { + eval(" + def test a + [*a, 3] + end + test [1, 2] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(assert_compiles("test [1, 2]"), @"[1, 2, 3]"); +} + +#[test] +fn test_splatarray_existing_array() { + eval(" + def foo v + [1, 2, v] + end + def test a + foo(*a) + end + test [3] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(assert_compiles("test [3]"), @"[1, 2, 3]"); +} + +#[test] +fn test_concattoarray() { + eval(" + def test(*a) + [1, 2, *a] + end + test 3 + "); + assert_contains_opcode("test", YARVINSN_concattoarray); + assert_snapshot!(assert_compiles("test 3"), @"[1, 2, 3]"); +} + +#[test] +fn test_definedivar() { + eval(" + def test + v0 = defined?(@a) + @a = nil + v1 = defined?(@a) + remove_instance_variable :@a + v2 = defined?(@a) + [v0, v1, v2] + end + test + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(assert_compiles("test"), @r#"[nil, "instance-variable", nil]"#); +} + +#[test] +fn test_setglobal_with_trace_var_exception() { + eval(r#" + def test + $a = 1 + rescue + "rescued" + end + trace_var(:$a) { raise } + test + "#); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(assert_compiles(r#"test"#), @r#""rescued""#); +} + +#[test] +fn test_getlocal_after_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_getlocal_after_instance_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + instance_eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_getlocal_after_module_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + Kernel.module_eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_getlocal_after_class_eval() { + assert_snapshot!(inspect(" + def test + a = 1 + Kernel.class_eval('a = 2') + a + end + test + test + "), @"2"); +} + +#[test] +fn test_setlocal() { + assert_snapshot!(inspect(" + def test(n) + m = n + m + end + test(3) + test(3) + "), @"3"); +} + +#[test] +fn test_return_nonparam_local() { + assert_snapshot!(inspect(" + def foo(a) + if false + x = nil + end + x + end + def test = foo(1) + test + test + "), @"nil"); +} + +#[test] +fn test_nonparam_local_nil_in_jit_call() { + assert_snapshot!(inspect(r#" + def f(a) + a ||= 1 + if false; b = 1; end + eval("-> { p 'x#{b}' }") + end + + 4.times.map { f(1).call } + "#), @r#"["x", "x", "x", "x"]"#); +} + +#[test] +fn test_kwargs_with_exit_and_local_invalidation() { + assert_snapshot!(inspect(r#" + def a(b:, c:) + if c == :b + return -> {} + end + Class # invalidate locals + + raise "c is :b!" if c == :b + end + + def test + # note opposite order of kwargs + a(c: :c, b: :b) + end + + 4.times { test } + :ok + "#), @":ok"); +} + +#[test] +fn test_kwargs_with_max_direct_send_arg_count() { + assert_snapshot!(inspect(" + def kwargs(five, six, a:, b:, c:, d:, e:, f:) + [a, b, c, d, five, six, e, f] + end + + 5.times.flat_map do + [ + kwargs(5, 6, d: 4, c: 3, a: 1, b: 2, e: 7, f: 8), + kwargs(5, 6, d: 4, c: 3, b: 2, a: 1, e: 7, f: 8) + ] + end.uniq + "), @"[[1, 2, 3, 4, 5, 6, 7, 8]]"); +} + +#[test] +fn test_setlocal_on_eval() { + assert_snapshot!(inspect(" + @b = binding + eval('a = 1', @b) + eval('a', @b) + "), @"1"); +} + +#[test] +fn test_optional_arguments() { + assert_snapshot!(inspect(" + def test(a, b = 2, c = 3) + [a, b, c] + end + [test(1), test(10, 20), test(100, 200, 300)] + "), @"[[1, 2, 3], [10, 20, 3], [100, 200, 300]]"); +} + +#[test] +fn test_optional_arguments_setlocal() { + assert_snapshot!(inspect(" + def test(a = (b = 2)) + [a, b] + end + [test, test(1)] + "), @"[[2, 2], [1, nil]]"); +} + +#[test] +fn test_optional_arguments_cyclic() { + assert_snapshot!(inspect(" + test = proc { |a=a| a } + [test.call, test.call(1)] + "), @"[nil, 1]"); +} + +#[test] +fn test_getblockparamproxy() { + eval(" + def test(&block) + 0.then(&block) + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(assert_compiles("test { 1 }"), @"1"); +} + +#[test] +fn test_getblockparamproxy_modified() { + eval(" + def test(&block) + b = block + 0.then(&block) + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(inspect("test { 1 }"), @"1"); +} + +#[test] +fn test_getblockparamproxy_modified_nested_block() { + eval(" + def test(&block) + proc do + b = block + 0.then(&block) + end + end + test { 1 }.call + "); + assert_snapshot!(inspect("test { 1 }.call"), @"1"); +} + +#[test] +fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(assert_compiles("test { 2 }"), @"2"); +} + +#[test] +fn test_getblockparam() { + eval(" + def test(&blk) + blk + end + test { 2 }.call + "); + assert_contains_opcode("test", YARVINSN_getblockparam); + assert_snapshot!(assert_compiles("test { 2 }.call"), @"2"); +} + +#[test] +fn test_setblockparam() { + eval(" + def test(&block) + block = proc { 3 } + blk = block + blk.call + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(assert_compiles("test { 1 }"), @"3"); +} + +#[test] +fn test_setblockparam_nested_block() { + eval(" + def test(&block) + proc do + block = proc { 3 } + blk = block + blk.call + end.call + end + test { 1 } + "); + assert_snapshot!(assert_compiles("test { 1 }"), @"3"); +} + +#[test] +fn test_getblockparamproxy_after_setblockparam() { + eval(" + def test(&block) + block = proc { 3 } + block.call + end + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(assert_compiles("test { 1 }"), @"3"); +} + +#[test] +fn test_getblockparam_used_twice_in_args() { + eval(" + def f(*args) = args + def test(&blk) + b = blk + f(*[1], blk) + blk + end + test {1}.call + "); + assert_contains_opcode("test", YARVINSN_getblockparam); + assert_snapshot!(assert_compiles("test {1}.call"), @"1"); +} + +#[test] +fn test_optimized_method_call_proc_call() { + eval(" + def test(p) + p.call(1) + end + test(proc { |x| x * 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test(proc { |x| x * 2 })"), @"2"); +} + +#[test] +fn test_optimized_method_call_proc_aref() { + eval(" + def test(p) + p[2] + end + test(proc { |x| x * 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(proc { |x| x * 2 })"), @"4"); +} + +#[test] +fn test_optimized_method_call_proc_yield() { + eval(" + def test(p) + p.yield(3) + end + test(proc { |x| x * 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test(proc { |x| x * 2 })"), @"6"); +} + +#[test] +fn test_optimized_method_call_proc_kw_splat() { + eval(" + def test(p, h) + p.call(**h) + end + test(proc { |**kw| kw[:a] + kw[:b] }, { a: 1, b: 2 }) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test(proc { |**kw| kw[:a] + kw[:b] }, { a: 1, b: 2 })"), @"3"); +} + +#[test] +fn test_optimized_method_call_proc_call_splat() { + assert_snapshot!(inspect(" + p = proc { |x| x + 1 } + def test(p) + ary = [42] + p.call(*ary) + end + test(p) + test(p) + "), @"43"); +} + +#[test] +fn test_optimized_method_call_proc_call_kwarg() { + assert_snapshot!(inspect(" + p = proc { |a:| a } + def test(p) + p.call(a: 1) + end + test(p) + test(p) + "), @"1"); +} + +#[test] +fn test_setlocal_on_eval_with_spill() { + assert_snapshot!(inspect(" + @b = binding + eval('a = 1; itself', @b) + eval('a', @b) + "), @"1"); +} + +#[test] +fn test_nested_local_access() { + assert_snapshot!(inspect(" + 1.times do |l2| + 1.times do |l1| + define_method(:test) do + l1 = 1 + l2 = 2 + l3 = 3 + [l1, l2, l3] + end + end + end + + test + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_send_with_local_written_by_blockiseq() { + assert_snapshot!(inspect(" + def test + l1 = nil + l2 = nil + tap do |_| + l1 = 1 + tap do |_| + l2 = 2 + end + end + + [l1, l2] + end + + test + test + "), @"[1, 2]"); +} + +#[test] +fn test_no_ep_escape_patch_point_after_send_does_not_repeat_send() { + eval(r#" + $send_count = 0 + + def test + captured = nil + tap do |_| + $send_count += 1 + -> { captured } if $send_count == 2 + end + $send_count + end + "#); + assert_contains_opcode("test", YARVINSN_send); + assert_snapshot!(assert_compiles_allowing_exits("[test, test, test]"), @"[1, 2, 3]"); +} + +#[test] +fn test_send_without_block() { + assert_snapshot!(inspect(" + def foo = 1 + def bar(a) = a - 1 + def baz(a, b) = a - b + + def test1 = foo + def test2 = bar(3) + def test3 = baz(4, 1) + + [test1, test2, test3] + "), @"[1, 2, 3]"); +} + +#[test] +fn test_send_with_six_args() { + assert_snapshot!(inspect(" + def foo(a1, a2, a3, a4, a5, a6) + [a1, a2, a3, a4, a5, a6] + end + + def test + foo(1, 2, 3, 4, 5, 6) + end + + test # profile send + test + "), @"[1, 2, 3, 4, 5, 6]"); +} + +#[test] +fn test_send_optional_arguments() { + assert_snapshot!(inspect(" + def test(a, b = 2) = [a, b] + def entry = [test(1), test(3, 4)] + entry + entry + "), @"[[1, 2], [3, 4]]"); +} + +#[test] +fn test_send_nil_block_arg() { + assert_snapshot!(inspect(" + def test = block_given? + def entry = test(&nil) + test + test + "), @"false"); +} + +#[test] +fn test_send_symbol_block_arg() { + assert_snapshot!(inspect(" + def test = [1, 2].map(&:to_s) + test + test + "), @r#"["1", "2"]"#); +} + +#[test] +fn test_send_variadic_with_block() { + assert_snapshot!(inspect(" + A = [1, 2, 3] + B = [\"a\", \"b\", \"c\"] + + def test + result = [] + A.zip(B) { |x, y| result << [x, y] } + result + end + + test; test + "), @r#"[[1, "a"], [2, "b"], [3, "c"]]"#); +} + +#[test] +fn test_send_kwarg_optional() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2) = [a, b] + def entry = test + entry + entry + "), @"[1, 2]"); +} + +#[test] +fn test_send_kwarg_optional_too_many() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2, c: 3, d: 4, e: 5, f: 6, g: 7, h: 8, i: 9, j: 10) = [a, b, c, d, e, f, g, h, i, j] + def entry = test + entry + entry + "), @"[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]"); +} + +#[test] +fn test_send_kwarg_required_and_optional() { + assert_snapshot!(inspect(" + def test(a:, b: 2) = [a, b] + def entry = test(a: 3) + entry + entry + "), @"[3, 2]"); +} + +#[test] +fn test_send_kwarg_to_hash() { + assert_snapshot!(inspect(" + def test(hash) = hash + def entry = test(a: 3) + entry + entry + "), @"{a: 3}"); +} + +#[test] +fn test_send_kwarg_to_ccall() { + assert_snapshot!(inspect(r#" + def test(s) = s.each_line(chomp: true).to_a + def entry = test(%(a\nb\nc)) + entry + entry + "#), @r#"["a", "b", "c"]"#); +} + +#[test] +fn test_send_kwarg_and_block_to_ccall() { + assert_snapshot!(inspect(r#" + def test(s) + a = [] + s.each_line(chomp: true) { |l| a << l } + a + end + def entry = test(%(a\nb\nc)) + entry + entry + "#), @r#"["a", "b", "c"]"#); +} + +#[test] +fn test_send_kwarg_with_too_many_args_to_c_call() { + assert_snapshot!(inspect(r#" + def test(a:, b:, c:, d:, e:) = sprintf("%s %s %s %s %s", a, b, c, d, kwargs: e) + def entry = test(e: :e, d: :d, c: :c, a: :a, b: :b) + entry + entry + "#), @r#""a b c d {kwargs: :e}""#); +} + +#[test] +fn test_send_kwsplat() { + assert_snapshot!(inspect(" + def test(a:) = a + def entry = test(**{a: 3}) + entry + entry + "), @"3"); +} + +#[test] +fn test_send_kwrest() { + assert_snapshot!(inspect(" + def test(**kwargs) = kwargs + def entry = test(a: 3) + entry + entry + "), @"{a: 3}"); +} + +#[test] +fn test_send_req_kwreq() { + assert_snapshot!(inspect(" + def test(a, c:) = [a, c] + def entry = test(1, c: 3) + entry + entry + "), @"[1, 3]"); +} + +#[test] +fn test_send_req_opt_kwreq() { + assert_snapshot!(inspect(" + def test(a, b = 2, c:) = [a, b, c] + def entry = [test(1, c: 3), test(-1, -2, c: -3)] + entry + entry + "), @"[[1, 2, 3], [-1, -2, -3]]"); +} + +#[test] +fn test_send_req_opt_kwreq_kwopt() { + assert_snapshot!(inspect(" + def test(a, b = 2, c:, d: 4) = [a, b, c, d] + def entry = [test(1, c: 3), test(-1, -2, d: -4, c: -3)] + entry + entry + "), @"[[1, 2, 3, 4], [-1, -2, -3, -4]]"); +} + +#[test] +fn test_send_unexpected_keyword() { + assert_snapshot!(inspect(" + def test(a: 1) = a*2 + def entry + test(z: 2) + rescue ArgumentError + :error + end + + entry + entry + "), @":error"); +} + +#[test] +fn test_pos_optional_with_maybe_too_many_args() { + assert_snapshot!(inspect(" + def target(a = 1, b = 2, c = 3, d = 4, e = 5, f:) = [a, b, c, d, e, f] + def test = [target(f: 6), target(10, 20, 30, f: 6), target(10, 20, 30, 40, 50, f: 60)] + test + test + "), @"[[1, 2, 3, 4, 5, 6], [10, 20, 30, 4, 5, 6], [10, 20, 30, 40, 50, 60]]"); +} + +#[test] +fn test_send_kwarg_partial_optional() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2, c: 3) = [a, b, c] + def entry = [test, test(b: 20), test(c: 30, a: 10)] + entry + entry + "), @"[[1, 2, 3], [1, 20, 3], [10, 2, 30]]"); +} + +#[test] +fn test_send_kwarg_optional_a_lot() { + assert_snapshot!(inspect(" + def test(a: 1, b: 2, c: 3, d: 4, e: 5, f: 6) = [a, b, c, d, e, f] + def entry = [test, test(d: 7, f: 9, e: 8), test(f: 12, e: 10, d: 8, c: 6, b: 4, a: 2)] + entry + entry + "), @"[[1, 2, 3, 4, 5, 6], [1, 2, 3, 7, 8, 9], [2, 4, 6, 8, 10, 12]]"); +} + +#[test] +fn test_send_kwarg_non_constant_default() { + assert_snapshot!(inspect(" + def make_default = 2 + def test(a: 1, b: make_default) = [a, b] + def entry = [test, test(a: 10)] + entry + entry + "), @"[[1, 2], [10, 2]]"); +} + +#[test] +fn test_send_kwarg_optional_static_with_side_exit() { + assert_snapshot!(inspect(" + def callee(a: 1, b: 2) + x = binding.local_variable_get(:a) + [a, b, x] + end + + def entry + callee(a: 10) + end + + entry + entry + "), @"[10, 2, 10]"); +} + +#[test] +fn test_send_hash_to_kwarg_only_method() { + assert_snapshot!(inspect(r#" + def callee(a:) = a + + def entry + callee({a: 1}) + rescue ArgumentError + "ArgumentError" + end + + entry + entry + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_send_hash_to_optional_kwarg_only_method() { + assert_snapshot!(inspect(r#" + def callee(a: nil) = a + + def entry + callee({a: 1}) + rescue ArgumentError + "ArgumentError" + end + + entry + entry + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_send_all_arg_types() { + assert_snapshot!(inspect(" + def test(a, b = :opt, c, d:, e: :kwo) = [a, b, c, d, e, block_given?] + def entry = test(:req, :post, d: :kwr) {} + entry + entry + "), @"[:req, :opt, :post, :kwr, :kwo, true]"); +} + +#[test] +fn test_send_ccall_variadic_with_different_receiver_classes() { + assert_snapshot!(inspect(r#" + def test(obj) = obj.start_with?("a") + [test("abc"), test(:abc)] + "#), @"[true, true]"); +} + +#[test] +fn test_forwardable_iseq() { + assert_snapshot!(inspect(" + def test(...) = 1 + test + test + "), @"1"); +} + +#[test] +fn test_sendforward() { + eval(" + def callee(a, b) = [a, b] + def test(...) = callee(...) + test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_sendforward); + assert_snapshot!(assert_compiles("test(1, 2)"), @"[1, 2]"); +} + +#[test] +fn test_iseq_with_optional_arguments() { + assert_snapshot!(inspect(" + def test(a, b = 2) = [a, b] + [test(1), test(3, 4)] + "), @"[[1, 2], [3, 4]]"); +} + +#[test] +fn test_invokesuper() { + assert_snapshot!(inspect(" + class Foo + def foo(a) = a + 1 + def bar(a) = a + 10 + end + + class Bar < Foo + def foo(a) = super(a) + 2 + def bar(a) = super + 20 + end + + bar = Bar.new + [bar.foo(3), bar.bar(30)] + "), @"[6, 60]"); +} + +#[test] +fn test_invokesuper_to_iseq() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + def test + B.new.foo + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "A"]"#); +} + +#[test] +fn test_invokesuper_with_args() { + assert_snapshot!(inspect(r#" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(x) + ["B", super(x) + 1] + end + end + + def test + B.new.foo(5) + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", 11]"#); +} + +#[test] +fn test_invokesuper_with_args_to_rest_param() { + assert_snapshot!(inspect(r#" + class A + def foo(x, *rest) + [x, rest] + end + end + + class B < A + def foo(x, y, z) + ["B", *super(x, y, z)] + end + end + + def test + B.new.foo("a", "b", "c") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "a", ["b", "c"]]"#); +} + +#[test] +fn test_invokesuper_with_block() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super { "from_block" }] + end + end + + def test + B.new.foo + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "from_block"]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_no_args() { + assert_snapshot!(inspect(r#" + class MyString < String + def length + ["MyString", super] + end + end + + def test + MyString.new("abc").length + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", 3]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_simple_args() { + assert_snapshot!(inspect(r#" + class MyString < String + def include?(other) + ["MyString", super(other)] + end + end + + def test + MyString.new("abc").include?("bc") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", true]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_with_optional_arg() { + assert_snapshot!(inspect(r#" + class MyString < String + def byteindex(needle, offset = 0) + ["MyString", super(needle, offset)] + end + end + + def test + MyString.new("hello world").byteindex("world") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", 6]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_varargs() { + assert_snapshot!(inspect(r#" + class MyString < String + def end_with?(str) + ["MyString", super(str)] + end + end + + def test + MyString.new("abc").end_with?("bc") + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["MyString", true]"#); +} + +#[test] +fn test_invokesuper_to_cfunc_with_too_many_args_exits() { + unsafe extern "C" fn test_six_args( + _self: VALUE, + a: VALUE, + b: VALUE, + c: VALUE, + d: VALUE, + e: VALUE, + f: VALUE, + ) -> VALUE { + unsafe { rb_ary_new_from_args(6, a, b, c, d, e, f) } + } + + with_rubyvm(|| { + let superclass = define_class("ZJITSixArgs", unsafe { rb_cObject }); + unsafe { + rb_define_method( + superclass, + c"six".as_ptr(), + Some(std::mem::transmute::< + unsafe extern "C" fn(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) -> VALUE, + unsafe extern "C" fn(VALUE) -> VALUE, + >(test_six_args)), + 6, + ); + } + }); + + assert_snapshot!(assert_compiles_allowing_exits(r#" + class ZJITSixArgsSubclass < ZJITSixArgs + def six(a, b, c, d, e, f) + super + end + end + + def test + ZJITSixArgsSubclass.new.six(1, 2, 3, 4, 5, 6) + end + + test + test + test + "#), @"[1, 2, 3, 4, 5, 6]"); +} + +#[test] +fn test_string_new_preserves_string_arg() { + assert_snapshot!(inspect(r#" + def test + str = "hello" + String.new(str) + :ok + end + + test + test + "#), @":ok"); +} + +#[test] +fn test_invokesuper_multilevel() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + class C < B + def foo + ["C", super] + end + end + + def test + C.new.foo + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["C", ["B", "A"]]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super] # should forward the block from caller + end + end + + def test + B.new.foo { "forwarded_block" } + end + + test # profile invokesuper + test # compile + run compiled code + "#), @r#"["B", "forwarded_block"]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly_with_args() { + assert_snapshot!(inspect(r#" + class A + def foo(x) + [x, (block_given? ? yield : "no_block")] + end + end + + class B < A + def foo(x) + ["B", super(x)] # explicit args, but block should still be forwarded + end + end + + def test + B.new.foo("arg_value") { "forwarded" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", ["arg_value", "forwarded"]]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly_no_block_given() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super] # no block given by caller + end + end + + def test + B.new.foo # called without a block + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "no_block"]"#); +} + +#[test] +fn test_invokesuper_forwards_block_implicitly_multilevel() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo + ["B", super] # forwards block to A + end + end + + class C < B + def foo + ["C", super] # forwards block to B, which forwards to A + end + end + + def test + C.new.foo { "deep_block" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["C", ["B", "deep_block"]]"#); +} + +#[test] +fn test_invokesuper_forwards_block_param() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no_block" + end + end + + class B < A + def foo(&block) + ["B", super] # should forward &block implicitly + end + end + + def test + B.new.foo { "block_param_forwarded" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "block_param_forwarded"]"#); +} + +#[test] +fn test_invokesuper_with_blockarg() { + assert_snapshot!(inspect(r#" + class A + def foo + block_given? ? yield : "no block" + end + end + + class B < A + def foo(&blk) + other_block = proc { "different block" } + ["B", super(&other_block)] + end + end + + def test + B.new.foo { "passed block" } + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "different block"]"#); +} + +#[test] +fn test_invokesuper_with_symbol_to_proc() { + assert_snapshot!(inspect(r#" + class A + def foo(items, &blk) + items.map(&blk) + end + end + + class B < A + def foo(items) + ["B", super(items, &:succ)] + end + end + + def test + B.new.foo([2, 4, 6]) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", [3, 5, 7]]"#); +} + +#[test] +fn test_invokesuper_with_splat() { + assert_snapshot!(inspect(r#" + class A + def foo(a, b, c) + a + b + c + end + end + + class B < A + def foo(*args) + ["B", super(*args)] + end + end + + def test + B.new.foo(1, 2, 3) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", 6]"#); +} + +#[test] +fn test_invokesuper_with_kwargs() { + assert_snapshot!(inspect(r#" + class A + def foo(x:, y:) + "x=#{x}, y=#{y}" + end + end + + class B < A + def foo(x:, y:) + ["B", super(x: x, y: y)] + end + end + + def test + B.new.foo(x: 1, y: 2) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "x=1, y=2"]"#); +} + +#[test] +fn test_invokesuper_with_kw_splat() { + assert_snapshot!(inspect(r#" + class A + def foo(x:, y:) + "x=#{x}, y=#{y}" + end + end + + class B < A + def foo(**kwargs) + ["B", super(**kwargs)] + end + end + + def test + B.new.foo(x: 1, y: 2) + end + + test # profile + test # compile + run compiled code + "#), @r#"["B", "x=1, y=2"]"#); +} + +#[test] +fn test_invokesuper_with_include() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + def test + B.new.foo + end + + test # profile invokesuper (super -> A#foo) + test # compile with super -> A#foo + + # Now include a module in B that defines foo - super should go to M#foo instead + module M + def foo + "M" + end + end + B.include(M) + + test # should call M#foo, not A#foo + "#), @r#"["B", "M"]"#); +} + +#[test] +fn test_invokesuper_with_prepend() { + assert_snapshot!(inspect(r#" + class A + def foo + "A" + end + end + + class B < A + def foo + ["B", super] + end + end + + def test + B.new.foo + end + + test # profile invokesuper (super -> A#foo) + test # compile with super -> A#foo + + # Now prepend a module that defines foo - super should go to M#foo instead + module M + def foo + "M" + end + end + A.prepend(M) + + test # should call M#foo, not A#foo + "#), @r#"["B", "M"]"#); +} + +#[test] +fn test_invokesuper_with_keyword_args() { + assert_snapshot!(inspect(r#" + class A + def foo(attributes = {}) + @attributes = attributes + end + end + + class B < A + def foo(content = '') + super(content: content) + end + end + + def test + B.new.foo("image data") + end + + test + test + "#), @r#"{content: "image data"}"#); +} + +#[test] +fn test_invokesuper_with_optional_keyword_args() { + assert_snapshot!(inspect(" + class Parent + def foo(a, b: 2, c: 3) = [a, b, c] + end + + class Child < Parent + def foo(a) = super(a) + end + + def test = Child.new.foo(1) + + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_invokesuperforward() { + assert_snapshot!(inspect(" + class A + def foo(a,b,c) = [a,b,c] + end + + class B < A + def foo(...) = super + end + + def test + B.new.foo(1, 2, 3) + end + + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_invokesuperforward_with_args_kwargs_and_block() { + assert_snapshot!(inspect(" + class A + def foo(*args, **kwargs, &block) + [args, kwargs, block&.call] + end + end + + class B < A + def foo(...) = super + end + + def test + B.new.foo(1, 2, x: 3) { 4 } + end + + test + test + "), @"[[1, 2], {x: 3}, 4]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default() { + assert_snapshot!(inspect(" + def dbl(x = 1) = x * 2 + + def foo(a: dbl, b: dbl(2), c: dbl(2 ** 3)) + [a, b, c] + end + + def test + [ + foo, + foo(a: 10), + foo(b: 20), + foo(c: 30), + foo(a: 10, b: 20, c: 30) + ] + end + + test + test + "), @"[[2, 4, 16], [10, 4, 16], [2, 20, 16], [2, 4, 30], [10, 20, 30]]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_not_evaluated_when_provided() { + assert_snapshot!(inspect(" + def foo(a: raise, b: raise, c: raise) + [a, b, c] + end + + def test + foo(a: 1, b: 2, c: 3) + end + + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_evaluated_when_not_provided() { + assert_snapshot!(inspect(r#" + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + "#), @r#"["a", "b", "c"]"#); +} + +#[test] +fn test_send_with_non_constant_keyword_default_jit_to_jit() { + assert_snapshot!(inspect(" + def make_default(x) = x * 2 + + def callee(a: make_default(1), b: make_default(2), c: make_default(3)) + [a, b, c] + end + + def caller_method + callee + end + + # Warm up callee first so it gets JITted + callee + callee + + # Now warm up caller - this creates JIT-to-JIT call + caller_method + caller_method + "), @"[2, 4, 6]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_side_exit() { + assert_snapshot!(inspect(" + def make_b = 2 + + def callee(a: 1, b: make_b, c: 3) + x = binding.local_variable_get(:a) + y = binding.local_variable_get(:b) + z = binding.local_variable_get(:c) + [x, y, z] + end + + def test + callee(a: 10, c: 30) + end + + test + test + "), @"[10, 2, 30]"); +} + +#[test] +fn test_send_with_non_constant_keyword_default_evaluation_order() { + assert_snapshot!(inspect(r#" + def log(x) + $order << x + x + end + + def foo(a: log("a"), b: log("b"), c: log("c")) + [a, b, c] + end + + def test + results = [] + + $order = [] + foo + results << $order.dup + + $order = [] + foo(a: "A") + results << $order.dup + + $order = [] + foo(b: "B") + results << $order.dup + + $order = [] + foo(c: "C") + results << $order.dup + + results + end + + test + test + "#), @r#"[["a", "b", "c"], ["b", "c"], ["a", "c"], ["a", "b"]]"#); +} + +#[test] +fn test_send_with_too_many_non_constant_keyword_defaults() { + assert_snapshot!(inspect(" + def many_kwargs( k1: 1, k2: 2, k3: 3, k4: 4, k5: 5, k6: 6, k7: 7, k8: 8, k9: 9, k10: 10, k11: 11, k12: 12, k13: 13, k14: 14, k15: 15, k16: 16, k17: 17, k18: 18, k19: 19, k20: 20, k21: 21, k22: 22, k23: 23, k24: 24, k25: 25, k26: 26, k27: 27, k28: 28, k29: 29, k30: 30, k31: 31, k32: 32, k33: 33, k34: k33 + 1) = k1 + k34 + def t = many_kwargs + t + t + "), @"35"); +} + +#[test] +fn test_invokebuiltin_delegate() { + assert_snapshot!(inspect(" + def test = [].clone(freeze: true) + r = test + r2 = test + [r2, r2.frozen?] + "), @"[[], true]"); +} + +#[test] +fn test_opt_plus_const() { + assert_snapshot!(inspect(" + def test = 1 + 2 + test # profile opt_plus + test + "), @"3"); +} + +#[test] +fn test_opt_plus_fixnum() { + assert_snapshot!(inspect(" + def test(a, b) = a + b + test(0, 1) # profile opt_plus + test(1, 2) + "), @"3"); +} + +#[test] +fn test_opt_plus_chain() { + assert_snapshot!(inspect(" + def test(a, b, c) = a + b + c + test(0, 1, 2) # profile opt_plus + test(1, 2, 3) + "), @"6"); +} + +#[test] +fn test_opt_plus_left_imm() { + assert_snapshot!(inspect(" + def test(a) = 1 + a + test(1) # profile opt_plus + test(2) + "), @"3"); +} + +#[test] +fn test_opt_plus_type_guard_exit() { + assert_snapshot!(inspect(" + def test(a) = 1 + a + test(1) # profile opt_plus + [test(2), test(2.0)] + "), @"[3, 3.0]"); +} + +#[test] +fn test_opt_plus_type_guard_exit_with_locals() { + assert_snapshot!(inspect(" + def test(a) + local = 3 + 1 + a + local + end + test(1) # profile opt_plus + [test(2), test(2.0)] + "), @"[6, 6.0]"); +} + +#[test] +fn test_opt_plus_type_guard_nested_exit() { + assert_snapshot!(inspect(" + def side_exit(n) = 1 + n + def jit_frame(n) = 1 + side_exit(n) + def entry(n) = jit_frame(n) + entry(2) # profile send + [entry(2), entry(2.0)] + "), @"[4, 4.0]"); +} + +#[test] +fn test_opt_plus_type_guard_nested_exit_with_locals() { + assert_snapshot!(inspect(" + def side_exit(n) + local = 2 + 1 + n + local + end + def jit_frame(n) + local = 3 + 1 + side_exit(n) + local + end + def entry(n) = jit_frame(n) + entry(2) # profile send + [entry(2), entry(2.0)] + "), @"[9, 9.0]"); +} + +#[test] +fn test_opt_minus() { + assert_snapshot!(inspect(" + def test(a, b) = a - b + test(2, 1) # profile opt_minus + test(6, 4) + "), @"2"); +} + +#[test] +fn test_opt_mult() { + assert_snapshot!(inspect(" + def test(a, b) = a * b + test(1, 2) # profile opt_mult + test(2, 3) + "), @"6"); +} + +#[test] +fn test_opt_mult_overflow() { + assert_snapshot!(inspect(" + def test(a, b) + a * b + end + test(1, 1) # profile opt_mult + + r1 = test(2, 3) + r2 = test(2, -3) + r3 = test(2 << 40, 2 << 41) + r4 = test(2 << 40, -2 << 41) + r5 = test(1 << 62, 1 << 62) + + [r1, r2, r3, r4, r5] + "), @"[6, -6, 9671406556917033397649408, -9671406556917033397649408, 21267647932558653966460912964485513216]"); +} + +#[test] +fn test_opt_eq() { + eval(" + def test(a, b) = a == b + test(0, 2) # profile opt_eq + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(assert_compiles("[test(1, 1), test(0, 1)]"), @"[true, false]"); +} + +#[test] +fn test_opt_eq_with_minus_one() { + eval(" + def test(a) = a == -1 + test(1) # profile opt_eq + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(assert_compiles("[test(0), test(-1)]"), @"[false, true]"); +} + +#[test] +fn test_opt_neq_dynamic() { + eval(" + def test(a, b) = a != b + test(0, 2) # profile opt_neq + "); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(assert_compiles("[test(1, 1), test(0, 1)]"), @"[false, true]"); +} + +#[test] +fn test_opt_neq_fixnum() { + assert_snapshot!(inspect(" + def test(a, b) = a != b + test(0, 2) # profile opt_neq + [test(1, 1), test(0, 1)] + "), @"[false, true]"); +} + +#[test] +fn test_opt_neq_string_nil() { + assert_snapshot!(inspect(r#" + def test(str) = str != nil + test("x") # profile opt_neq + [test("x"), test(nil)] + "#), @"[true, false]"); +} + +#[test] +fn test_opt_neq_string_same_operand() { + assert_snapshot!(inspect(r#" + def test(s) = s != s + test("x") # profile opt_neq + [test("x"), test("y")] + "#), @"[false, false]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_neq_string_distinct_literals() { + assert_snapshot!(inspect(r#" + def test = "a" != "b" + test # profile opt_neq + [test, test] + "#), @"[true, true]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_neq_string_one_side_known_literal() { + assert_snapshot!(inspect(r#" + def test(s) = "a" != s + test("a") # profile opt_neq + [test("a"), test("b")] + "#), @"[false, true]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_neq_string_distinct_objects() { + assert_snapshot!(inspect(r#" + def test(s, t) = s != t + test("x", "x") # profile opt_neq + [test("x", "x"), test("x", "y")] + "#), @"[false, true]"); + assert_contains_opcode("test", YARVINSN_opt_neq); +} + +#[test] +fn test_opt_eq_string_same_operand() { + assert_snapshot!(inspect(r#" + def test(s) = s == s + test("x") # profile opt_eq + [test("x"), test("y")] + "#), @"[true, true]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eq_string_distinct_literals() { + assert_snapshot!(inspect(r#" + def test = "a" == "b" + test # profile opt_eq + [test, test] + "#), @"[false, false]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eq_string_one_side_known_literal() { + assert_snapshot!(inspect(r#" + def test(s) = "a" == s + test("a") # profile opt_eq + [test("a"), test("b")] + "#), @"[true, false]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eq_string_distinct_objects() { + assert_snapshot!(inspect(r#" + def test(s, t) = s == t + test("x", "x") # profile opt_eq + [test("x", "x"), test("x", "y")] + "#), @"[true, false]"); + assert_contains_opcode("test", YARVINSN_opt_eq); +} + +#[test] +fn test_opt_eqq_string_same_operand() { + assert_snapshot!(inspect(r#" + def test(s) = s === s + test("x") # profile opt_send_without_block + [test("x"), test("y")] + "#), @"[true, true]"); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); +} + +#[test] +fn test_opt_lt() { + eval(" + def test(a, b) = a < b + test(2, 3) # profile opt_lt + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[true, false, false]"); +} + +#[test] +fn test_opt_lt_with_literal_lhs() { + eval(" + def test(n) = 2 < n + test(2) # profile opt_lt + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(assert_compiles("[test(1), test(2), test(3)]"), @"[false, false, true]"); +} + +#[test] +fn test_opt_le() { + eval(" + def test(a, b) = a <= b + test(2, 3) # profile opt_le + "); + assert_contains_opcode("test", YARVINSN_opt_le); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[true, true, false]"); +} + +#[test] +fn test_opt_gt() { + eval(" + def test(a, b) = a > b + test(2, 3) # profile opt_gt + "); + assert_contains_opcode("test", YARVINSN_opt_gt); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[false, false, true]"); +} + +#[test] +fn test_opt_empty_p() { + eval(" + def test(x) = x.empty? + "); + assert_contains_opcode("test", YARVINSN_opt_empty_p); + assert_snapshot!(assert_compiles_allowing_exits("[test([1]), test(\"1\"), test({})]"), @"[false, false, true]"); +} + +#[test] +fn test_opt_succ() { + eval(" + def test(obj) = obj.succ + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(assert_compiles_allowing_exits(r#"[test(-1), test("A")]"#), @r#"[0, "B"]"#); +} + +#[test] +fn test_opt_and() { + eval(" + def test(x, y) = x & y + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(assert_compiles_allowing_exits("[test(0b1101, 3), test([3, 2, 1, 4], [8, 1, 2, 3])]"), @"[1, [3, 2, 1]]"); +} + +#[test] +fn test_opt_or() { + eval(" + def test(x, y) = x | y + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(assert_compiles_allowing_exits("[test(0b1000, 3), test([3, 2, 1], [1, 2, 3])]"), @"[11, [3, 2, 1]]"); +} + +#[test] +fn test_fixnum_and() { + eval(" + def test(a, b) = a & b + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(assert_compiles(" + [ + test(5, 3), + test(0b011, 0b110), + test(-0b011, 0b110) + ] + "), @"[1, 2, 4]"); +} + +#[test] +fn test_fixnum_and_side_exit() { + eval(" + def test(a, b) = a & b + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(assert_compiles_allowing_exits(" + [ + test(2, 2), + test(0b011, 0b110), + test(true, false) + ] + "), @"[2, 2, false]"); +} + +#[test] +fn test_fixnum_or() { + eval(" + def test(a, b) = a | b + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(assert_compiles(" + [ + test(5, 3), + test(1, 2), + test(1, -4) + ] + "), @"[7, 3, -3]"); +} + +#[test] +fn test_fixnum_or_side_exit() { + eval(" + def test(a, b) = a | b + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(assert_compiles_allowing_exits(" + [ + test(1, 2), + test(2, 2), + test(true, false) + ] + "), @"[3, 2, true]"); +} + +#[test] +fn test_fixnum_xor() { + assert_snapshot!(inspect(" + def test(a, b) = a ^ b + [ + test(5, 3), + test(-5, 3), + test(1, 2) + ] + "), @"[6, -8, 3]"); +} + +#[test] +fn test_fixnum_xor_side_exit() { + assert_snapshot!(inspect(" + def test(a, b) = a ^ b + [ + test(5, 3), + test(5, 3), + test(true, false) + ] + "), @"[6, 6, true]"); +} + +#[test] +fn test_fixnum_mul() { + eval(" + C = 3 + def test(n) = C * n + test(4) + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_mult); + assert_snapshot!(assert_compiles("test(4)"), @"12"); +} + +#[test] +fn test_fixnum_div() { + eval(" + C = 48 + def test(n) = C / n + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles("test(4)"), @"12"); +} + +#[test] +fn test_fixnum_floor() { + eval(" + C = 3 + def test(n) = C / n + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles("test(4)"), @"0"); +} + +#[test] +fn test_fixnum_mod() { + eval(" + def test(a, b) = a % b + test(13, 4) # profile opt_mod + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(assert_compiles("[test(13, 4), test(13, 13), test(5, 7)]"), @"[1, 0, 5]"); +} + +#[test] +fn test_fixnum_mod_negative() { + eval(" + def test(a, b) = a % b + test(7, 3) # profile opt_mod + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(assert_compiles("[test(-7, 3), test(7, -3), test(-7, -3)]"), @"[2, -2, -1]"); +} + +#[test] +fn test_fixnum_mod_by_zero() { + eval(" + def test(a, b) = a % b rescue :zero_div + test(13, 4) # profile opt_mod + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(assert_compiles_allowing_exits("test(13, 0)"), @":zero_div"); +} + +#[test] +fn test_fixnum_div_min_by_neg_one() { + // FIXNUM_MIN / -1 overflows to a Bignum: the JIT must side exit, not return a mistyped Fixnum. + eval(" + def test(a, b) = a / b + test(10, 3) # profile opt_div + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles_allowing_exits("test(-4611686018427387904, -1)"), @"4611686018427387904"); +} + +#[test] +fn test_fixnum_div_overflow_propagation() { + // The div must side exit before its Bignum result reaches the specialized (a / b) & 1 op. + eval(" + def test(a, b) = (a / b) & 1 + test(10, 3) # profile opt_div + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles_allowing_exits("test(-4611686018427387904, -1)"), @"0"); +} + +#[test] +fn test_fixnum_div_by_neg_one_is_fine() { + // x / -1 (x != FIXNUM_MIN) is a normal Fixnum and must NOT trip the overflow guard. + eval(" + def test(a, b) = a / b + test(10, 3) # profile opt_div + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles("test(10, -1)"), @"-10"); +} + +#[test] +fn test_opt_not() { + eval(" + def test(obj) = !obj + "); + assert_contains_opcode("test", YARVINSN_opt_not); + assert_snapshot!(assert_compiles_allowing_exits("[test(nil), test(false), test(0)]"), @"[true, true, false]"); +} + +#[test] +fn test_opt_regexpmatch2() { + eval(" + def test(haystack) = /needle/ =~ haystack + "); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(assert_compiles(r#"[test("kneedle"), test("")]"#), @"[1, nil]"); +} + +#[test] +fn test_opt_ge() { + eval(" + def test(a, b) = a >= b + test(2, 3) # profile opt_ge + "); + assert_contains_opcode("test", YARVINSN_opt_ge); + assert_snapshot!(assert_compiles("[test(0, 1), test(0, 0), test(1, 0)]"), @"[false, true, true]"); +} + +#[test] +fn test_opt_new_does_not_push_frame() { + eval(" + class Foo + attr_reader :backtrace + def initialize + @backtrace = caller + end + end + def test = Foo.new + test + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles(" + foo = test + foo.backtrace.find { |frame| frame.include?('Class#new') } + "), @"nil"); +} + +#[test] +fn test_opt_new_with_redefined() { + eval(r#" + class Foo + def self.new = "foo" + def initialize = raise("unreachable") + end + def test = Foo.new + test + "#); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles(r#"test"#), @r#""foo""#); +} + +#[test] +fn test_opt_new_invalidate_new() { + eval(r#" + class Foo; end + def test = Foo.new + test + "#); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles(r#" + result = [test.class.name] + def Foo.new = "foo" + result << test + "#), @r#"["Foo", "foo"]"#); +} + +#[test] +fn test_opt_newarray_send_include_p() { + eval(" + def test(x) + [:y, 1, Object.new].include?(x) + end + test(1) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles("[test(1), test(\"n\")]"), @"[true, false]"); +} + +#[test] +fn test_opt_newarray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) ? :true : :false + end + end + def test(x) + [:y, 1, Object.new].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(" + def test(x) + [:y, 1, Object.new].include?(x) + end + test(1) + [test(1), test(\"n\")] + "), @"[:true, :false]"); +} + +#[test] +fn test_opt_duparray_send_include_p() { + eval(" + def test(x) + [:y, 1].include?(x) + end + test(1) + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(assert_compiles("[test(1), test(\"n\")]"), @"[true, false]"); +} + +#[test] +fn test_opt_duparray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) ? :true : :false + end + end + def test(x) + [:y, 1].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(assert_compiles_allowing_exits(" + def test(x) + [:y, 1].include?(x) + end + test(1) + [test(1), test(\"n\")] + "), @"[:true, :false]"); +} + +#[test] +fn test_opt_newarray_send_pack() { + eval(r#" + def test(num) + [num].pack('C') + end + test(65) + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles(r#" + [test(65), test(66), test(67)] + "#), @r#"["A", "B", "C"]"#); +} + +#[test] +fn test_opt_newarray_send_pack_redefined() { + eval(r#" + class Array + alias_method :old_pack, :pack + def pack(fmt, buffer: nil) + "override:#{old_pack(fmt, buffer: buffer)}" + end + end + def test(num) + [num].pack('C') + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(r#" + [test(65), test(66), test(67)] + "#), @r#"["override:A", "override:B", "override:C"]"#); +} + +#[test] +fn test_opt_newarray_send_pack_buffer() { + eval(r#" + def test(num, buffer) + [num].pack('C', buffer:) + end + test(65, "") + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles(r#" + buf = "" + [test(65, buf), test(66, buf), test(67, buf), buf] + "#), @r#"["ABC", "ABC", "ABC", "ABC"]"#); +} + +#[test] +fn test_opt_newarray_send_pack_buffer_redefined() { + eval(r#" + class Array + alias_method :old_pack, :pack + def pack(fmt, buffer: nil) + old_pack(fmt, buffer: buffer) + "b" + end + end + def test(num, buffer) + [num].pack('C', buffer:) + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(r#" + def test(num, buffer) + [num].pack('C', buffer:) + end + buf = "" + test(65, buf) + buf = "" + [test(65, buf), buf] + "#), @r#"["b", "A"]"#); +} + +#[test] +fn test_opt_newarray_send_hash() { + eval(" + def test(x) + [1, 2, x].hash + end + test(20) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles("test(20).class"), @"Integer"); +} + +#[test] +fn test_opt_newarray_send_hash_redefined() { + eval(" + Array.class_eval { def hash = 42 } + def test(x) + [1, 2, x].hash + end + test(20) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits("test(20)"), @"42"); +} + +#[test] +fn test_opt_newarray_send_max() { + eval(" + def test(a,b) = [a,b].max + test(10, 20) + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles("[test(10, 20), test(40, 30)]"), @"[20, 40]"); +} + +#[test] +fn test_opt_newarray_send_max_redefined() { + eval(" + class Array + alias_method :old_max, :max + def max + old_max * 2 + end + end + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(assert_compiles_allowing_exits(" + def test(a,b) = [a,b].max + test(15, 30) + [test(15, 30), test(45, 35)] + "), @"[60, 90]"); +} + +#[test] +fn test_new_hash_empty() { + eval(" + def test = {} + test + "); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles("test"), @"{}"); +} + +#[test] +fn test_new_hash_nonempty() { + eval(r#" + def test + key = "key" + value = "value" + num = 42 + result = 100 + {key => value, num => result} + end + test + "#); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles(r#"test"#), @r#"{"key" => "value", 42 => 100}"#); +} + +#[test] +fn test_new_hash_single_key_value() { + eval(r#" + def test = {"key" => "value"} + test + "#); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles(r#"test"#), @r#"{"key" => "value"}"#); +} + +#[test] +fn test_new_hash_with_computation() { + eval(r#" + def test(a, b) + {"sum" => a + b, "product" => a * b} + end + test(2, 3) + "#); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(assert_compiles(r#"test(2, 3)"#), @r#"{"sum" => 5, "product" => 6}"#); +} + +#[test] +fn test_new_hash_with_user_defined_hash_method() { + assert_snapshot!(inspect(r#" + class CustomKey + attr_reader :val + def initialize(val) + @val = val + end + def hash + @val.hash + end + def eql?(other) + other.is_a?(CustomKey) && @val == other.val + end + end + def test + key = CustomKey.new("key") + hash = {key => "value"} + hash[key] == "value" + end + test + test + "#), @"true"); +} + +#[test] +fn test_new_hash_with_user_hash_method_exception() { + assert_snapshot!(inspect(r#" + class BadKey + def hash + raise "Hash method failed!" + end + end + def test + key = BadKey.new + {key => "value"} + end + begin + test + rescue => e + e.class + end + begin + test + rescue => e + e.class + end + "#), @"RuntimeError"); +} + +#[test] +fn test_new_hash_with_user_eql_method_exception() { + assert_snapshot!(inspect(r#" + class BadKey + def hash + 42 + end + def eql?(other) + raise "Eql method failed!" + end + end + def test + key1 = BadKey.new + key2 = BadKey.new + {key1 => "value1", key2 => "value2"} + end + begin + test + rescue => e + e.class + end + begin + test + rescue => e + e.class + end + "#), @"RuntimeError"); +} + +#[test] +fn test_opt_hash_freeze() { + eval(" + def test = {}.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(assert_compiles(" + result = [test] + class Hash + def freeze = 5 + end + result << test + "), @"[{}, 5]"); +} + +#[test] +fn test_opt_hash_freeze_rewritten() { + eval(" + class Hash + def freeze = 5 + end + def test = {}.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_opt_aset_hash() { + eval(" + def test(h, k, v) + h[k] = v + end + test({}, :key, 42) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("h = {}; test(h, :key, 42); h[:key]"), @"42"); +} + +#[test] +fn test_opt_aset_hash_returns_value() { + assert_snapshot!(inspect(" + def test(h, k, v) + h[k] = v + end + test({}, :key, 100) + test({}, :key, 100) + "), @"100"); +} + +#[test] +fn test_opt_aset_hash_string_key() { + assert_snapshot!(inspect(r#" + def test(h, k, v) + h[k] = v + end + h = {} + test(h, "foo", "bar") + test(h, "foo", "bar") + h["foo"] + "#), @r#""bar""#); +} + +#[test] +fn test_opt_aset_hash_subclass() { + assert_snapshot!(inspect(" + class MyHash < Hash; end + def test(h, k, v) + h[k] = v + end + h = MyHash.new + test(h, :key, 42) + test(h, :key, 42) + h[:key] + "), @"42"); +} + +#[test] +fn test_opt_aset_hash_too_few_args() { + assert_snapshot!(inspect(r#" + def test(h) + h.[]= 123 + rescue ArgumentError + "ArgumentError" + end + test({}) + test({}) + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_opt_aset_hash_too_many_args() { + assert_snapshot!(inspect(r#" + def test(h) + h[:a, :b] = :c + rescue ArgumentError + "ArgumentError" + end + test({}) + test({}) + "#), @r#""ArgumentError""#); +} + +#[test] +fn test_opt_ary_freeze() { + eval(" + def test = [].freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(assert_compiles(" + result = [test] + class Array + def freeze = 5 + end + result << test + "), @"[[], 5]"); +} + +#[test] +fn test_opt_ary_freeze_rewritten() { + eval(" + class Array + def freeze = 5 + end + def test = [].freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_opt_str_freeze() { + eval(" + def test = ''.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(assert_compiles(r#" + result = [test] + class String + def freeze = 5 + end + result << test + "#), @r#"["", 5]"#); +} + +#[test] +fn test_opt_str_freeze_rewritten() { + eval(" + class String + def freeze = 5 + end + def test = ''.freeze + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_opt_str_uminus() { + eval(" + def test = -'' + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(assert_compiles(r#" + result = [test] + class String + def -@ = 5 + end + result << test + "#), @r#"["", 5]"#); +} + +#[test] +fn test_opt_str_uminus_rewritten() { + eval(" + class String + def -@ = 5 + end + def test = -'' + test + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(assert_compiles_allowing_exits("test"), @"5"); +} + +#[test] +fn test_new_array_empty() { + eval(" + def test = [] + test + "); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(assert_compiles("test"), @"[]"); +} + +#[test] +fn test_new_array_nonempty() { + assert_snapshot!(inspect(" + def a = 5 + def test = [a] + test + test + "), @"[5]"); +} + +#[test] +fn test_new_array_order() { + assert_snapshot!(inspect(" + def a = 3 + def b = 2 + def c = 1 + def test = [a, b, c] + test + test + "), @"[3, 2, 1]"); +} + +#[test] +fn test_array_dup() { + assert_snapshot!(inspect(" + def test = [1,2,3] + test + test + "), @"[1, 2, 3]"); +} + +#[test] +fn test_array_fixnum_aref() { + eval(" + def test(x) = [1,2,3][x] + test(2) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(2)"), @"3"); +} + +#[test] +fn test_array_fixnum_aref_negative_index() { + eval(" + def test(x) = [1,2,3][x] + test(-1) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(-1)"), @"3"); +} + +#[test] +fn test_array_fixnum_aref_out_of_bounds_positive() { + eval(" + def test(x) = [1,2,3][x] + test(10) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles_allowing_exits("test(10)"), @"nil"); +} + +#[test] +fn test_array_fixnum_aref_out_of_bounds_negative() { + eval(" + def test(x) = [1,2,3][x] + test(-10) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles_allowing_exits("test(-10)"), @"nil"); +} + +#[test] +fn test_array_fixnum_aref_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, idx) = arr[idx] + test(MyArray[1,2,3], 2) + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(assert_compiles("test(MyArray[1,2,3], 2)"), @"3"); +} + +#[test] +fn test_array_aref_non_fixnum_index() { + assert_snapshot!(inspect(r#" + def test(arr, idx) = arr[idx] + test([1,2,3], 1) + test([1,2,3], 1) + begin + test([1,2,3], "1") + rescue => e + e.class + end + "#), @"TypeError"); +} + +#[test] +fn test_array_fixnum_aset() { + eval(" + def test(arr, idx) + arr[idx] = 7 + end + test([1,2,3], 2) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("arr = [1,2,3]; test(arr, 2); arr"), @"[1, 2, 7]"); +} + +#[test] +fn test_array_fixnum_aset_returns_value() { + eval(" + def test(arr, idx) + arr[idx] = 7 + end + test([1,2,3], 2) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("test([1,2,3], 2)"), @"7"); +} + +#[test] +fn test_array_fixnum_aset_out_of_bounds() { + assert_snapshot!(inspect(" + def test(arr) + arr[5] = 7 + end + arr = [1,2,3] + test(arr) + arr = [1,2,3] + test(arr) + arr + "), @"[1, 2, 3, nil, nil, 7]"); +} + +#[test] +fn test_array_fixnum_aset_negative_index() { + assert_snapshot!(inspect(" + def test(arr) + arr[-1] = 7 + end + arr = [1,2,3] + test(arr) + arr = [1,2,3] + test(arr) + arr + "), @"[1, 2, 7]"); +} + +#[test] +fn test_array_fixnum_aset_shared() { + assert_snapshot!(inspect(" + def test(arr, idx, val) + arr[idx] = val + end + arr = (0..50).to_a + test(arr, 0, -1) + test(arr, 1, -2) + shared = arr[10, 20] + test(shared, 0, 999) + [arr[10], shared[0], arr[0], arr[1]] + "), @"[10, 999, -1, -2]"); +} + +#[test] +fn test_array_fixnum_aset_frozen() { + assert_snapshot!(inspect(" + def test(arr, idx, val) + arr[idx] = val + end + arr = [1,2,3] + test(arr, 1, 9) + test(arr, 1, 9) + arr.freeze + begin + test(arr, 1, 9) + rescue => e + e.class + end + "), @"FrozenError"); +} + +#[test] +fn test_array_fixnum_aset_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, idx) + arr[idx] = 7 + end + test(MyArray.new, 0) + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(assert_compiles("arr = MyArray.new; test(arr, 0); arr[0]"), @"7"); +} + +#[test] +fn test_array_aset_non_fixnum_index() { + assert_snapshot!(inspect(r#" + def test(arr, idx) + arr[idx] = 7 + end + test([1,2,3], 0) + test([1,2,3], 0) + begin + test([1,2,3], "0") + rescue => e + e.class + end + "#), @"TypeError"); +} + +#[test] +fn test_empty_array_pop() { + assert_snapshot!(inspect(" + def test(arr) = arr.pop + test([]) + test([]) + "), @"nil"); +} + +#[test] +fn test_array_pop_no_arg() { + assert_snapshot!(inspect(" + def test(arr) = arr.pop + test([32, 33, 42]) + test([32, 33, 42]) + "), @"42"); +} + +#[test] +fn test_array_pop_arg() { + assert_snapshot!(inspect(" + def test(arr) = arr.pop(2) + test([32, 33, 42]) + test([32, 33, 42]) + "), @"[33, 42]"); +} + +#[test] +fn test_new_range_inclusive() { + assert_snapshot!(inspect(" + def test(a, b) = a..b + test(1, 5) + test(1, 5) + "), @"1..5"); +} + +#[test] +fn test_new_range_exclusive() { + assert_snapshot!(inspect(" + def test(a, b) = a...b + test(1, 5) + test(1, 5) + "), @"1...5"); +} + +#[test] +fn test_new_range_with_literal() { + assert_snapshot!(inspect(" + def test(n) = n..10 + test(3) + test(3) + "), @"3..10"); +} + +#[test] +fn test_new_range_fixnum_both_literals_inclusive() { + eval(" + def test() + a = 2 + (1..a) + end + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test; test"), @"1..2"); +} + +#[test] +fn test_new_range_fixnum_both_literals_exclusive() { + eval(" + def test() + a = 2 + (1...a) + end + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test; test"), @"1...2"); +} + +#[test] +fn test_new_range_fixnum_low_literal_inclusive() { + eval(" + def test(a) = (1..a) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"1..3"); +} + +#[test] +fn test_new_range_fixnum_low_literal_exclusive() { + eval(" + def test(a) = (1...a) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"1...3"); +} + +#[test] +fn test_new_range_fixnum_high_literal_inclusive() { + eval(" + def test(a) = (a..10) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"3..10"); +} + +#[test] +fn test_new_range_fixnum_high_literal_exclusive() { + eval(" + def test(a) = (a...10) + "); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(assert_compiles("test(2); test(3)"), @"3...10"); +} + +#[test] +fn test_if() { + assert_snapshot!(inspect(" + def test(n) + if n < 5 + 0 + end + end + test(3) + [test(3), test(7)] + "), @"[0, nil]"); +} + +#[test] +fn test_if_else() { + assert_snapshot!(inspect(" + def test(n) + if n < 5 + 0 + else + 1 + end + end + test(3) + [test(3), test(7)] + "), @"[0, 1]"); +} + +#[test] +fn test_if_else_params() { + assert_snapshot!(inspect(" + def test(n, a, b) + if n < 5 + a + else + b + end + end + test(3, 1, 2) + [test(3, 1, 2), test(7, 10, 20)] + "), @"[1, 20]"); +} + +#[test] +fn test_if_else_nested() { + assert_snapshot!(inspect(" + def test(a, b, c, d, e) + if 2 < a + if a < 4 + b + else + c + end + else + if a < 0 + d + else + e + end + end + end + test(-1, 1, 2, 3, 4) + [ + test(-1, 1, 2, 3, 4), + test( 0, 5, 6, 7, 8), + test( 3, 9, 10, 11, 12), + test( 5, 13, 14, 15, 16), + ] + "), @"[3, 8, 9, 14]"); +} + +#[test] +fn test_if_else_chained() { + assert_snapshot!(inspect(" + def test(a) + (if 2 < a then 1 else 2 end) + (if a < 4 then 10 else 20 end) + end + test(0) + [test(0), test(3), test(5)] + "), @"[12, 11, 21]"); +} + +#[test] +fn test_if_elsif_else() { + assert_snapshot!(inspect(" + def test(n) + if n < 5 + 0 + elsif 8 < n + 1 + else + 2 + end + end + test(3) + [test(3), test(7), test(9)] + "), @"[0, 2, 1]"); +} + +#[test] +fn test_ternary_operator() { + assert_snapshot!(inspect(" + def test(n, a, b) + n < 5 ? a : b + end + test(3, 1, 2) + [test(3, 1, 2), test(7, 10, 20)] + "), @"[1, 20]"); +} + +#[test] +fn test_ternary_operator_nested() { + assert_snapshot!(inspect(" + def test(n, a, b) + (n < 5 ? a : b) + 1 + end + test(3, 1, 2) + [test(3, 1, 2), test(7, 10, 20)] + "), @"[2, 21]"); +} + +#[test] +fn test_while_loop() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + while i < n + i = i + 1 + end + i + end + test(10) + test(10) + "), @"10"); +} + +#[test] +fn test_while_loop_chain() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + while i < n + i = i + 1 + end + while i < n * 10 + i = i * 3 + end + i + end + test(5) + [test(5), test(10)] + "), @"[135, 270]"); +} + +#[test] +fn test_while_loop_nested() { + assert_snapshot!(inspect(" + def test(n, m) + i = 0 + while i < n + j = 0 + while j < m + j += 2 + end + i += j + end + i + end + test(0, 0) + [test(0, 0), test(1, 3), test(10, 5)] + "), @"[0, 4, 12]"); +} + +#[test] +fn test_while_loop_if_else() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + while i < n + if n >= 10 + return -1 + else + i = i + 1 + end + end + i + end + test(9) + [test(9), test(10)] + "), @"[9, -1]"); +} + +#[test] +fn test_if_while_loop() { + assert_snapshot!(inspect(" + def test(n) + i = 0 + if n < 10 + while i < n + i += 1 + end + else + while i < n + i += 3 + end + end + i + end + test(9) + [test(9), test(10)] + "), @"[9, 12]"); +} + +#[test] +fn test_live_reg_past_ccall() { + assert_snapshot!(inspect(" + def callee = 1 + def test = callee + callee + test + test + "), @"2"); +} + +#[test] +fn test_method_call() { + assert_snapshot!(inspect(" + def callee(a, b) + a - b + end + def test + callee(4, 2) + 10 + end + test + test + "), @"12"); +} + +#[test] +fn test_recursive_fact() { + assert_snapshot!(inspect(" + def fact(n) + if n == 0 + return 1 + end + return n * fact(n-1) + end + fact(0) + [fact(0), fact(3), fact(6)] + "), @"[1, 6, 720]"); +} + +#[test] +fn test_recursive_fib() { + assert_snapshot!(inspect(" + def fib(n) + if n < 2 + return n + end + return fib(n-1) + fib(n-2) + end + fib(0) + [fib(0), fib(3), fib(4)] + "), @"[0, 2, 3]"); +} + +#[test] +fn test_spilled_basic_block_args() { + assert_snapshot!(inspect(" + def test(n1, n2) + n3 = 3 + n4 = 4 + n5 = 5 + n6 = 6 + n7 = 7 + n8 = 8 + n9 = 9 + n10 = 10 + if n1 < n2 + n1 + n2 + n3 + n4 + n5 + n6 + n7 + n8 + n9 + n10 + end + end + test(1, 2) + test(1, 2) + "), @"55"); +} + +#[test] +fn test_putself() { + assert_snapshot!(inspect(" + class Integer + def minus(a) + self - a + end + end + 5.minus(2) + 5.minus(2) + "), @"3"); +} + +#[test] +fn test_getinstancevariable_nil() { + assert_snapshot!(inspect(" + def test() = @foo + test() + test() + "), @"nil"); +} + +#[test] +fn test_getinstancevariable() { + assert_snapshot!(inspect(" + @foo = 3 + def test() = @foo + test() + test() + "), @"3"); +} + +#[test] +fn test_getinstancevariable_miss() { + assert_snapshot!(inspect(" + class C + def foo + @foo + end + def foo_then_bar + @foo = 1 + @bar = 2 + end + def bar_then_foo + @bar = 3 + @foo = 4 + end + end + o1 = C.new + o1.foo_then_bar + result = [] + result << o1.foo + result << o1.foo + o2 = C.new + o2.bar_then_foo + result << o2.foo + result + "), @"[1, 1, 4]"); +} + +#[test] +fn test_setinstancevariable() { + assert_snapshot!(inspect(" + def test() = @foo = 1 + test() + test() + @foo + "), @"1"); +} + +#[test] +fn test_getclassvariable() { + assert_snapshot!(inspect(" + class Foo + def self.test = @@x + end + Foo.class_variable_set(:@@x, 42) + Foo.test() + Foo.test() + "), @"42"); +} + +#[test] +fn test_getclassvariable_raises() { + assert_snapshot!(inspect(r#" + class Foo + def self.test = @@x + end + begin + Foo.test + Foo.test + rescue NameError => e + e.message + end + "#), @r#""uninitialized class variable @@x in Foo""#); +} + +#[test] +fn test_setclassvariable() { + assert_snapshot!(inspect(" + class Foo + def self.test = @@x = 42 + end + Foo.test() + Foo.test() + Foo.class_variable_get(:@@x) + "), @"42"); +} + +#[test] +fn test_setclassvariable_raises() { + assert_snapshot!(inspect(r#" + class Foo + def self.test = @@x = 42 + freeze + end + begin + Foo.test + Foo.test + rescue FrozenError => e + e.message + end + "#), @r#""can't modify frozen Class: Foo""#); +} + +#[test] +fn test_attr_reader() { + eval(" + class C + attr_reader :foo + def initialize + @foo = 4 + end + end + def test(c) = c.foo + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[4, 4]"); +} + +#[test] +fn test_attr_accessor_getivar() { + eval(" + class C + attr_accessor :foo + def initialize + @foo = 4 + end + end + def test(c) = c.foo + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[4, 4]"); +} + +#[test] +fn test_getivar_t_data_then_string() { + // This is a regression test for a type confusion miscomp where + // we end up reading the fields object using an offset off of a + // string, assuming that it has a the same layout as a T_DATA object. + // At the time of writing the fields object of strings are stored + // in a global table, out-of-line of each string. + // The string and the thread end up sharing one shape ID. + set_call_threshold(2); + eval(r#" + module GetThousand + def test = @var1000 + end + class Thread + include GetThousand + end + class String + include GetThousand + end + OBJ = Thread.new { } + OBJ.join + STR = +'' + (0..1000).each do |i| + ivar_name = :"@var#{i}" + OBJ.instance_variable_set(ivar_name, i) + STR.instance_variable_set(ivar_name, i) + end + OBJ.test; OBJ.test # profile and compile for Thread (T_DATA) + "#); + assert_snapshot!(assert_compiles_allowing_exits("[STR.test, STR.test]"), @"[1000, 1000]"); +} + +#[test] +fn test_getivar_t_object_then_string() { + // This test construct an object and a string that have the same set of ivars. + // They wouldn't share the same shape ID, though, and we rely on this fact in + // our guards. + set_call_threshold(2); + eval(r#" + module GetThousand + def test = @var1000 + end + class MyObject + include GetThousand + end + class String + include GetThousand + end + OBJ = MyObject.new + STR = +'' + (0..1000).each do |i| + ivar_name = :"@var#{i}" + OBJ.instance_variable_set(ivar_name, i) + STR.instance_variable_set(ivar_name, i) + end + OBJ.test; OBJ.test # profile and compile for MyObject + "#); + assert_snapshot!(assert_compiles_allowing_exits("[STR.test, STR.test]"), @"[1000, 1000]"); +} + +#[test] +fn test_getivar_t_class_then_string() { + // This is a regression test for a type confusion miscomp where + // we end up reading the fields object using an offset off of a + // string, assuming that it has a the same layout as a T_CLASS object. + // At the time of writing the fields object of strings are stored + // in a global table, out-of-line of each string. + // The string and the class end up sharing one shape ID. + set_call_threshold(2); + eval(r#" + module GetThousand + def test = @var1000 + end + class MyClass + extend GetThousand + end + class String + include GetThousand + end + STR = +'' + (0..1000).each do |i| + ivar_name = :"@var#{i}" + MyClass.instance_variable_set(ivar_name, i) + STR.instance_variable_set(ivar_name, i) + end + p MyClass.test; p MyClass.test # profile and compile for MyClass + p STR.test + "#); + assert_snapshot!(assert_compiles_allowing_exits("[STR.test, STR.test]"), @"[1000, 1000]"); +} + + +#[test] +fn test_attr_accessor_setivar() { + eval(" + class C + attr_accessor :foo + def initialize + @foo = 4 + end + end + def test(c) + c.foo = 5 + c.foo + end + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[5, 5]"); +} + +#[test] +fn test_attr_writer() { + eval(" + class C + attr_writer :foo + def initialize + @foo = 4 + end + def get_foo = @foo + end + def test(c) + c.foo = 5 + c.get_foo + end + test(C.new) + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("c = C.new; [test(c), test(c)]"), @"[5, 5]"); +} + +#[test] +fn test_getconstant() { + eval(" + class Foo + CONST = 1 + end + def test(klass) + klass::CONST + end + test(Foo) + "); + assert_contains_opcode("test", YARVINSN_getconstant); + assert_snapshot!(assert_compiles("test(Foo)"), @"1"); +} + +#[test] +fn test_expandarray_no_splat() { + eval(" + def test(o) + a, b = o + [a, b] + end + test [3, 4] + "); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(assert_compiles("test [3, 4]"), @"[3, 4]"); +} + +#[test] +fn test_expandarray_splat() { + eval(" + def test(o) + a, *b = o + [a, b] + end + test [3, 4] + "); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(assert_compiles_allowing_exits("test [3, 4]"), @"[3, [4]]"); +} + +#[test] +fn test_expandarray_splat_post() { + eval(" + def test(o) + a, *b, c = o + [a, b, c] + end + test [3, 4, 5] + "); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(assert_compiles_allowing_exits("test [3, 4, 5]"), @"[3, [4], 5]"); +} + +#[test] +fn test_constant_invalidation() { + eval(" + class C; end + def test = C + test + test + C = 123 + "); + assert_contains_opcode("test", YARVINSN_opt_getconstant_path); + assert_snapshot!(assert_compiles("test"), @"123"); +} + +#[test] +fn test_constant_path_invalidation() { + eval(" + module A + module B; end + end + module Foo + C = 'Foo::C' + end + A::B = Foo + def test = A::B::C + "); + assert_contains_opcode("test", YARVINSN_opt_getconstant_path); + assert_snapshot!(assert_compiles(r#" + module A + module B; end + end + module Foo + C = "Foo::C" + end + module Bar + C = "Bar::C" + end + A::B = Foo + def test = A::B::C + result = [] + result << test + result << test + A::B = Bar + result << test + result + "#), @r#"["Foo::C", "Foo::C", "Bar::C"]"#); +} + +#[test] +fn test_dupn() { + eval(" + def test(array) = (array[1, 2] ||= :rhs) + test([1, 1]) + "); + assert_contains_opcode("test", YARVINSN_dupn); + assert_snapshot!(assert_compiles_allowing_exits(" + one = [1, 1] + start_empty = [] + [test(one), one, test(start_empty), start_empty] + "), @"[[1], [1, 1], :rhs, [nil, :rhs]]"); +} + +#[test] +fn test_bop_invalidation() { + assert_snapshot!(inspect(r#" + def test + eval("class Integer; def +(_) = 100; end") + 1 + 2 + end + test + test + "#), @"100"); +} + +#[test] +fn test_defined_with_defined_values() { + eval(" + class Foo; end + def bar; end + $ruby = 1 + def test = [defined?(Foo), defined?(bar), defined?($ruby)] + test + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles("test"), @r#"["constant", "method", "global-variable"]"#); +} + +#[test] +fn test_defined_with_undefined_values() { + eval(" + def test = [defined?(FooUndef), defined?(bar_undef), defined?($ruby_undef)] + test + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles("test"), @"[nil, nil, nil]"); +} + +#[test] +fn test_defined_with_method_call() { + eval(r#" + def test = [defined?("x".reverse(1)), defined?("x".reverse(1).reverse)] + test + "#); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles(r#"test"#), @r#"["method", nil]"#); +} + +#[test] +fn test_defined_method_raise() { + assert_snapshot!(inspect(r#" + class C + def assert_equal expected, actual + if expected != actual + raise "NO" + end + end + def test_defined_method + assert_equal(nil, defined?("x".reverse(1).reverse)) + end + end + c = C.new + result = [] + result << c.test_defined_method + result << c.test_defined_method + result << c.test_defined_method + result + "#), @"[nil, nil, nil]"); +} + +#[test] +fn test_defined_yield() { + eval(" + def test = defined?(yield) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(assert_compiles("[test, test, test{}]"), @r#"[nil, nil, "yield"]"#); +} + +#[test] +fn test_defined_yield_from_block() { + assert_snapshot!(inspect(" + def test + yield_self { yield_self { defined?(yield) } } + end + [test, test, test{}] + "), @r#"[nil, nil, "yield"]"#); +} + +#[test] +fn test_block_given_p() { + assert_snapshot!(inspect(" + def test = block_given? + [test, test, test{}] + "), @"[false, false, true]"); +} + +#[test] +fn test_block_given_p_from_block() { + assert_snapshot!(inspect(" + def test + yield_self { yield_self { block_given? } } + end + [test, test, test{}] + "), @"[false, false, true]"); +} + +#[test] +fn test_invokeblock_without_block_after_jit_call() { + assert_snapshot!(inspect(r#" + def test(*arr, &b) + arr.class + yield + end + test { } + begin + test + rescue => e + e.message + end + "#), @r#""no block given (yield)""#); +} + +#[test] +fn test_putspecialobject_vm_core_and_cbase() { + eval(" + def test + alias bar test + 10 + end + test + "); + assert_contains_opcode("test", YARVINSN_putspecialobject); + assert_snapshot!(assert_compiles("bar"), @"10"); +} + +#[test] +fn test_putspecialobject_const_base() { + assert_snapshot!(inspect(" + Foo = 1 + def test = Foo + test + test + "), @"1"); +} + +#[test] +fn test_branchnil() { + eval(" + def test(x) + x&.succ + end + test(0) + "); + assert_contains_opcode("test", YARVINSN_branchnil); + assert_snapshot!(assert_compiles("[test(1), test(nil)]"), @"[2, nil]"); +} + +#[test] +fn test_nil_nil() { + eval(" + def test = nil.nil? + test + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test"), @"true"); +} + +#[test] +fn test_non_nil_nil() { + eval(" + def test = 1.nil? + test + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test"), @"false"); +} + +#[test] +fn test_getspecial_last_match() { + eval(r#" + def test(str) + str =~ /hello/ + $& + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""hello""#); +} + +#[test] +fn test_getspecial_match_pre() { + eval(r#" + def test(str) + str =~ /world/ + $` + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""hello ""#); +} + +#[test] +fn test_getspecial_match_post() { + eval(r#" + def test(str) + str =~ /hello/ + $' + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#"" world""#); +} + +#[test] +fn test_getspecial_match_last_group() { + eval(r#" + def test(str) + str =~ /(hello) (world)/ + $+ + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""world""#); +} + +#[test] +fn test_getspecial_numbered_match_1() { + eval(r#" + def test(str) + str =~ /(hello) (world)/ + $1 + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""hello""#); +} + +#[test] +fn test_getspecial_numbered_match_2() { + eval(r#" + def test(str) + str =~ /(hello) (world)/ + $2 + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @r#""world""#); +} + +#[test] +fn test_getspecial_numbered_match_nonexistent() { + eval(r#" + def test(str) + str =~ /(hello)/ + $2 + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @"nil"); +} + +#[test] +fn test_getspecial_no_match() { + eval(r#" + def test(str) + str =~ /xyz/ + $& + end + test("hello world") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("hello world")"#), @"nil"); +} + +#[test] +fn test_getspecial_complex_pattern() { + eval(r#" + def test(str) + str =~ /(\d+)/ + $1 + end + test("abc123def") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("abc123def")"#), @r#""123""#); +} + +#[test] +fn test_getspecial_multiple_groups() { + eval(r#" + def test(str) + str =~ /(\d+)-(\d+)/ + $2 + end + test("123-456") + "#); + assert_contains_opcode("test", YARVINSN_getspecial); + assert_snapshot!(assert_compiles(r#"test("123-456")"#), @r#""456""#); +} + +// In a JIT-to-JIT call, the callee's cfp->jit_return is published at entry. +// Putting $& as the first C call in the callee exercises CFP_ZJIT_FRAME before +// gen_save_pc_for_gc has a chance to update the entry JITFrame. +#[test] +fn test_getspecial_symbol_in_jit_to_jit_callee() { + eval(r#" + def callee = $& + def caller_method = callee + + # Warm up callee so it JITs + callee + callee + + caller_method + caller_method + "#); + assert_contains_opcode("callee", YARVINSN_getspecial); + assert_snapshot!(assert_compiles("caller_method"), @"nil"); +} + +// Same JIT-to-JIT setup, exercising gen_getspecial_number ($N). +#[test] +fn test_getspecial_number_in_jit_to_jit_callee() { + eval(r#" + def callee = $1 + def caller_method = callee + + callee + callee + + caller_method + caller_method + "#); + assert_contains_opcode("callee", YARVINSN_getspecial); + assert_snapshot!(assert_compiles("caller_method"), @"nil"); +} + +#[test] +fn test_profile_under_nested_jit_call() { + assert_snapshot!(inspect(" + def profile + 1 + 2 + end + def jit_call(flag) + if flag + profile + end + end + def entry(flag) + jit_call(flag) + end + [entry(false), entry(false), entry(true)] + "), @"[nil, nil, 3]"); +} + +#[test] +fn test_bop_redefined() { + assert_snapshot!(inspect(" + def test + 1 + 2 + end + test + [test, Integer.class_eval { def +(_) = 100 }, test] + "), @"[3, :+, 100]"); +} + +#[test] +fn test_bop_redefined_with_adjacent_patch_points() { + assert_snapshot!(inspect(" + def test + 1 + 2 + 3 + 4 + 5 + end + test + [test, Integer.class_eval { def +(_) = 100 }, test] + "), @"[15, :+, 100]"); +} + +#[test] +fn test_method_redefined_with_top_self() { + assert_snapshot!(inspect(r#" + def foo + "original" + end + def test = foo + test; test + result1 = test + def foo + "redefined" + end + result2 = test + [result1, result2] + "#), @r#"["original", "redefined"]"#); +} + +#[test] +fn test_method_redefined_with_module() { + assert_snapshot!(inspect(r#" + module Foo + def self.foo = "original" + end + def test = Foo.foo + test + result1 = test + def Foo.foo = "redefined" + result2 = test + [result1, result2] + "#), @r#"["original", "redefined"]"#); +} + +#[test] +fn test_module_name_with_guard_passes() { + assert_snapshot!(inspect(r#" + def test(mod) + mod.name + end + test(String) + test(Integer) + "#), @r#""Integer""#); +} + +#[test] +fn test_module_name_with_guard_side_exit() { + assert_snapshot!(inspect(r#" + class MyClass + def name = "Bar" + end + def test(mod) + mod.name + end + results = [] + results << test(String) + results << test(Integer) + results << test(MyClass.new) + results + "#), @r#"["String", "Integer", "Bar"]"#); +} + +#[test] +fn test_objtostring_calls_to_s_on_non_strings() { + assert_snapshot!(inspect(r##" + results = [] + class Foo + def to_s + "foo" + end + end + def test(str) + "#{str}" + end + results << test(Foo.new) + results << test(Foo.new) + results + "##), @r#"["foo", "foo"]"#); +} + +#[test] +fn test_objtostring_rewrite_does_not_call_to_s_on_strings() { + assert_snapshot!(inspect(r##" + results = [] + class String + def to_s + "bad" + end + end + def test(foo) + "#{foo}" + end + results << test("foo") + results << test("foo") + results + "##), @r#"["foo", "foo"]"#); +} + +#[test] +fn test_objtostring_rewrite_does_not_call_to_s_on_string_subclasses() { + assert_snapshot!(inspect(r##" + results = [] + class StringSubclass < String + def to_s + "bad" + end + end + foo = StringSubclass.new("foo") + def test(str) + "#{str}" + end + results << test(foo) + results << test(foo) + results + "##), @r#"["foo", "foo"]"#); +} + +#[test] +fn test_objtostring_profiled_string_fastpath() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test('foo'); test('foo') + "##), @r#""foo""#); +} + +#[test] +fn test_objtostring_profiled_string_subclass_fastpath() { + assert_snapshot!(inspect(r##" + class MyString < String; end + def test(str) + "#{str}" + end + foo = MyString.new("foo") + test(foo); test(foo) + "##), @r#""foo""#); +} + +#[test] +fn test_objtostring_profiled_string_fastpath_exits_on_nonstring() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test('foo') + test(1) + "##), @r#""1""#); +} + +#[test] +fn test_objtostring_profiled_nonstring_calls_to_s() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test([1,2,3]); + test([1,2,3]); + "##), @r#""[1, 2, 3]""#); +} + +#[test] +fn test_objtostring_profiled_nonstring_guard_exits_when_string() { + assert_snapshot!(inspect(r##" + def test(str) + "#{str}" + end + test([1,2,3]); + test('foo'); + "##), @r#""foo""#); +} + +#[test] +fn test_string_bytesize_with_guard() { + assert_snapshot!(inspect(" + def test(str) + str.bytesize + end + test('hello') + test('world') + "), @"5"); +} + +#[test] +fn test_string_bytesize_multibyte() { + assert_snapshot!(inspect(r#" + def test(s) + s.bytesize + end + test("💎") + test("💎") + "#), @"4"); +} + +#[test] +fn test_nil_value_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(nil) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(nil)"), @"true"); +} + +#[test] +fn test_nil_value_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(nil) + test(nil) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(1)"), @"false"); +} + +#[test] +fn test_true_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(true) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(true)"), @"false"); +} + +#[test] +fn test_true_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(true) + test(true) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_false_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(false) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(false)"), @"false"); +} + +#[test] +fn test_false_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(false) + test(false) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_integer_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(1) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(2)"), @"false"); +} + +#[test] +fn test_integer_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(1) + test(2) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_float_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(1.0) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(2.0)"), @"false"); +} + +#[test] +fn test_float_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(1.0) + test(2.0) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_symbol_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(:foo) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles("test(:bar)"), @"false"); +} + +#[test] +fn test_symbol_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(:foo) + test(:bar) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_class_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(String) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(Integer)"), @"false"); +} + +#[test] +fn test_class_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(String) + test(Integer) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_module_nil_opt_with_guard() { + eval(" + def test(val) = val.nil? + test(Enumerable) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(Kernel)"), @"false"); +} + +#[test] +fn test_module_nil_opt_with_guard_side_exit() { + eval(" + def test(val) = val.nil? + test(Enumerable) + test(Kernel) + "); + assert_contains_opcode("test", YARVINSN_opt_nil_p); + assert_snapshot!(assert_compiles_allowing_exits("test(nil)"), @"true"); +} + +#[test] +fn test_basic_object_guard_works_with_immediate() { + assert_snapshot!(inspect(" + class Foo; end + def test(val) = val.class + test(Foo.new) + test(Foo.new) + test(nil) + "), @"NilClass"); +} + +#[test] +fn test_basic_object_guard_works_with_false() { + assert_snapshot!(inspect(" + class Foo; end + def test(val) = val.class + test(Foo.new) + test(Foo.new) + test(false) + "), @"FalseClass"); +} + +#[test] +fn test_string_concat() { + eval(r##" + def test = "#{1}#{2}#{3}" + test + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(assert_compiles(r##"test"##), @r#""123""#); +} + +#[test] +fn test_string_concat_empty() { + eval(r##" + def test = "#{}" + test + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(assert_compiles(r##"test"##), @r#""""#); +} + +#[test] +fn test_regexp_interpolation() { + eval(r##" + def test = /#{1}#{2}#{3}/ + test + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(assert_compiles(r##"test"##), @"/123/"); +} + +#[test] +fn test_new_range_non_leaf() { + assert_snapshot!(inspect(" + def jit_entry(v) = make_range_then_exit(v) + def make_range_then_exit(v) + range = (v..1) + super rescue range + end + jit_entry(0) + jit_entry(0) + jit_entry(0/1r) + "), @"(0/1)..1"); +} + +#[test] +fn test_raise_in_second_argument() { + assert_snapshot!(inspect(" + def write(hash, key) + hash[key] = raise rescue true + hash + end + write({}, :warmup) + write({}, :ok) + "), @"{ok: true}"); +} + +#[test] +fn test_struct_set() { + assert_snapshot!(inspect(" + C = Struct.new(:foo).new(1) + def test + C.foo = Object.new + 42 + end + r = [test, test] + C.freeze + r << begin + test + rescue FrozenError + :frozen_error + end + "), @"[42, 42, :frozen_error]"); +} + +#[test] +fn test_opt_case_dispatch() { + eval(" + def test(x) + case x + when :foo + true + else + false + end + end + test(:warmup) + "); + assert_contains_opcode("test", YARVINSN_opt_case_dispatch); + assert_snapshot!(assert_compiles("[test(:foo), test(1)]"), @"[true, false]"); +} + +#[test] +fn test_checkmatch_case() { + eval(r#" + def test(o) + case o + in Integer + 1 + else + 2 + end + end + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(assert_compiles(r#"[test(1), test(2), test("3")]"#), @"[1, 1, 2]"); +} + +#[test] +fn test_checkmatch_case_splat_array() { + eval(r#" + def test(o) + case o + when *[1, 2] + 1 + else + 2 + end + end + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(assert_compiles("[test(1), test(2), test(3)]"), @"[1, 1, 2]"); +} + +#[test] +fn test_checkmatch_when_splat_array() { + eval(r#" + def test + case + when *[1, 2] + 1 + else + 2 + end + end + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(assert_compiles("[test, test]"), @"[1, 1]"); +} + +#[test] +fn test_checkmatch_rescue() { + // Rescue behavior is tested functionally here. It still side-exits because + // JIT exception handling is not supported yet. + eval(r#" + def test + begin + raise TypeError + rescue TypeError + 1 + end + end + "#); + assert_snapshot!(assert_compiles("[test, test]"), @"[1, 1]"); +} + +#[test] +fn test_checkmatch_rescue_splat_array() { + eval(r#" + def test + begin + raise TypeError + rescue *[TypeError, ArgumentError] + 1 + end + end + "#); + assert_snapshot!(assert_compiles("[test, test]"), @"[1, 1]"); +} + +#[test] +fn test_stack_overflow() { + assert_snapshot!(inspect(" + def recurse(n) + return if n == 0 + recurse(n-1) + nil + end + recurse(2) + recurse(2) + begin + recurse(20_000) + rescue SystemStackError + end + "), @"nil"); +} + +#[test] +fn test_invokeblock() { + eval(" + def test + yield + end + test { 41 } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles("test { 42 }"), @"42"); +} + +#[test] +fn test_invokeblock_with_args() { + eval(" + def test(x, y) + yield x, y + end + test(1, 2) { |a, b| a + b } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles("test(1, 2) { |a, b| a + b }"), @"3"); +} + +#[test] +fn test_invokeblock_no_block_given() { + eval(" + def test + yield rescue :error + end + test { } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles("test"), @":error"); +} + +#[test] +fn test_invokeblock_multiple_yields() { + eval(" + def test + yield 1 + yield 2 + yield 3 + end + test { |x| x } + "); + assert_contains_opcode("test", YARVINSN_invokeblock); + assert_snapshot!(assert_compiles(" + results = [] + test { |x| results << x } + results + "), @"[1, 2, 3]"); +} + +#[test] +fn test_invokeblock_ifunc_map() { + eval(" + class MyList + include Enumerable + def each + yield 1 + yield 2 + yield 3 + end + end + def test = MyList.new.map { |x| x * 2 } + test + "); + assert_snapshot!(assert_compiles("test"), @"[2, 4, 6]"); +} + +#[test] +fn test_ccall_variadic_with_multiple_args() { + eval(" + def test + a = [] + a.push(1, 2, 3) + a + end + test + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test"), @"[1, 2, 3]"); +} + +#[test] +fn test_ccall_variadic_with_no_args() { + eval(" + def test + a = [1] + a.push + end + test + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test"), @"[1]"); +} + +#[test] +fn test_ccall_variadic_with_no_args_causing_argument_error() { + eval(" + def test + format + rescue ArgumentError + :error + end + test + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(assert_compiles("test"), @":error"); +} + +#[test] +fn test_allocating_in_hir_c_method_is() { + eval(" + def a(f) = test(f) + def test(f) = (f.new if f) + def second = third + def third = nil + a(nil) + a(nil) + class Foo + def self.new = :k + end + second + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(assert_compiles_allowing_exits("a(Foo)"), @":k"); +} + +#[test] +fn test_singleton_class_invalidation_annotated_ccall() { + assert_snapshot!(inspect(" + def define_singleton(obj, define) + if define + [nil].reverse_each do + class << obj + def ==(_) + true + end + end + end + end + false + end + def test(define) + obj = BasicObject.new + obj == define_singleton(obj, define) + end + result = [] + result << test(false) + result << test(true) + result + "), @"[false, true]"); +} + +#[test] +fn test_singleton_class_invalidation_optimized_variadic_ccall() { + assert_snapshot!(inspect(" + def define_singleton(arr, define) + if define + [nil].reverse_each do + class << arr + def push(x) + super(x * 1000) + end + end + end + end + 1 + end + def test(define) + arr = [] + val = define_singleton(arr, define) + arr.push(val) + arr[0] + end + result = [] + result << test(false) + result << test(true) + result + "), @"[1, 1000]"); +} + +#[test] +fn test_is_a_string_special_case() { + assert_snapshot!(inspect(r#" + def test(x) + x.is_a?(String) + end + test("foo") + [test("bar"), test(1), test(false), test(:foo), test([]), test({})] + "#), @"[true, false, false, false, false, false]"); +} + +#[test] +fn test_is_a_array_special_case() { + assert_snapshot!(inspect(" + def test(x) + x.is_a?(Array) + end + test([]) + [test([1,2,3]), test([]), test(1), test(false), test(:foo), test('foo'), test({})] + "), @"[true, true, false, false, false, false, false]"); +} + +#[test] +fn test_is_a_hash_special_case() { + assert_snapshot!(inspect(" + def test(x) + x.is_a?(Hash) + end + test({}) + [test({:a => 'b'}), test({}), test(1), test(false), test(:foo), test([]), test('foo')] + "), @"[true, true, false, false, false, false, false]"); +} + +#[test] +fn test_is_a_hash_subclass() { + assert_snapshot!(inspect(" + class MyHash < Hash + end + def test(x) + x.is_a?(Hash) + end + test({}) + test(MyHash.new) + "), @"true"); +} + +#[test] +fn test_is_a_normal_case() { + assert_snapshot!(inspect(r#" + class MyClass + end + def test(x) + x.is_a?(MyClass) + end + test("a") + [test(MyClass.new), test("a")] + "#), @"[true, false]"); +} + +#[test] +fn test_fixnum_div_zero() { + eval(" + def test(n) + n / 0 + rescue ZeroDivisionError => e + e.message + end + test(0) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(assert_compiles_allowing_exits(r#"test(0)"#), @r#""divided by 0""#); +} + +#[test] +fn test_invokesuper_with_local_written_by_blockiseq() { + assert_snapshot!(inspect(r#" + class A + def foo = "A" + end + class B < A + def foo + x = nil + [nil].each do |_| + x = super + end + x + end + end + def test = B.new.foo + test + test + "#), @r#""A""#); +} + +#[test] +fn test_max_iseq_versions() { + let max_versions = max_iseq_versions(); + eval(&format!(" + TEST = -1 + def test = TEST + + # compile and invalidate MAX+1 times + i = 0 + while i < {max_versions} + 1 + test; test # compile a version + + Object.send(:remove_const, :TEST) + TEST = i + + i += 1 + end + ")); + + // It should not exceed MAX_ISEQ_VERSIONS + let iseq = get_method_iseq("self", "test"); + let payload = get_or_create_iseq_payload(iseq); + assert_eq!(payload.versions.len(), max_iseq_versions()); + + // The last call should not discard the JIT code + assert!(matches!(unsafe { payload.versions.last().unwrap().as_ref() }.status, IseqStatus::Compiled(_))); +} + +#[test] +fn test_optional_arguments_side_exit() { + assert_snapshot!(inspect(" + def test(a = (def foo = nil)) = a + test + [test, (undef :foo), test(1)] + "), @"[:foo, nil, 1]"); +} + +#[test] +fn test_call_a_forwardable_method() { + assert_snapshot!(inspect(" + def test_root = forwardable + def forwardable(...) = Array.[](...) + test_root + test_root + "), @"[]"); +} + +#[test] +fn test_send_on_heap_object_in_spilled_arg() { + assert_snapshot!(inspect(" + def entry(a1, a2, a3, a4, a5, a6, a7, a8, a9) + a9.itself.class + end + entry(1, 2, 3, 4, 5, 6, 7, 8, {}) + entry(1, 2, 3, 4, 5, 6, 7, 8, {}) + "), @"Hash"); +} + +#[test] +fn test_send_splat() { + assert_snapshot!(inspect(" + def test(a, b) = [a, b] + def entry(arr) = test(*arr) + entry([1, 2]) + entry([1, 2]) + "), @"[1, 2]"); +} + +#[test] +fn test_send_kwarg() { + assert_snapshot!(inspect(" + def test(a:, b:) = [a, b] + def entry = test(b: 2, a: 1) + entry + entry + "), @"[1, 2]"); +} + +#[test] +fn test_spilled_method_args() { + assert_snapshot!(inspect(" + def foo(n1, n2, n3, n4, n5, n6, n7, n8, n9, n10) + n1 + n2 + n3 + n4 + n5 + n6 + n7 + n8 + n9 + n10 + end + def test + foo(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) + end + test + test + "), @"55"); +} + +#[test] +fn test_spilled_method_args_first_and_last() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8,n9) = n1+n9 + a(2,0,0,0,0,0,0,0,-1) + a(2,0,0,0,0,0,0,0,-1) + "), @"1"); +} + +#[test] +fn test_spilled_method_args_last() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8) = n8 + a(1,1,1,1,1,1,1,0) + a(1,1,1,1,1,1,1,0) + "), @"0"); +} + +#[test] +fn test_spilled_method_args_self() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8) = self + a(1,0,0,0,0,0,0,0).to_s + a(1,0,0,0,0,0,0,0).to_s + "), @r#""main""#); +} + +#[test] +fn test_spilled_param_new_array() { + assert_snapshot!(inspect(" + def a(n1,n2,n3,n4,n5,n6,n7,n8) = [n8] + a(0,0,0,0,0,0,0, :ok) + a(0,0,0,0,0,0,0, :ok) + "), @"[:ok]"); +} + +#[test] +fn test_forty_param_method() { + assert_snapshot!(inspect(" + def foo(_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,_,n40) = n40 + foo(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1) + foo(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1) + "), @"1"); +} + +#[test] +fn test_toplevel_local_after_eval() { + assert_snapshot!(inspect(" + a = 1 + b = 2 + eval('b = 3') + c = 4 + [a, b, c] + "), @"[1, 3, 4]"); +} + +#[test] +fn test_send_exit_with_uninitialized_locals() { + assert_snapshot!(inspect(" + def entry(init) + function_stub_exit(init) + end + + def function_stub_exit(init) + uninitialized_local = 1 if init + uninitialized_local + end + + entry(true) + entry(false) + "), @"nil"); +} + +#[test] +fn test_invokebuiltin_dir_glob() { + assert_snapshot!(inspect(r#" + def test = Dir.glob(".") + test + test + "#), @r#"["."]"#); +} + +#[test] +fn test_profiled_fact() { + assert_snapshot!(inspect(" + def fact(n) + if n == 0 + return 1 + end + return n * fact(n-1) + end + fact(1) + [fact(0), fact(3), fact(6)] + "), @"[1, 6, 720]"); +} + +#[test] +fn test_profiled_fib() { + assert_snapshot!(inspect(" + def fib(n) + if n < 2 + return n + end + return fib(n-1) + fib(n-2) + end + fib(3) + [fib(0), fib(3), fib(4)] + "), @"[0, 2, 3]"); +} + +#[test] +fn test_single_ractor_mode_invalidation() { + assert_snapshot!(inspect(r#" + C = Object.new + + def test + C + rescue Ractor::IsolationError + "errored but not crashed" + end + + test + test + + Ractor.new { + test + }.value + "#), @r#""errored but not crashed""#); +} + +#[test] +fn test_ivar_attr_reader_optimization_with_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + class << self + attr_accessor :bar + + def get_bar + bar + rescue Ractor::IsolationError + 42 + end + end + end + + Foo.bar = [] + + def test + Foo.get_bar + end + + test + test + + Ractor.new { test }.value + "), @"42"); +} + +#[test] +fn test_ivar_get_with_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + def self.set_bar + @bar = [] + end + + def self.bar + @bar + rescue Ractor::IsolationError + 42 + end + end + + Foo.set_bar + + def test + Foo.bar + end + + test + test + + Ractor.new { test }.value + "), @"42"); +} + +#[test] +fn test_ivar_get_with_already_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + def self.set_bar + @bar = [] + end + + def self.bar + @bar + rescue Ractor::IsolationError + 42 + end + end + + Foo.set_bar + r = Ractor.new { + Ractor.receive + Foo.bar + } + + Foo.bar + Foo.bar + + r << :go + r.value + "), @"42"); +} + +#[test] +fn test_ivar_set_with_multi_ractor_mode() { + assert_snapshot!(inspect(" + class Foo + def self.bar + _foo = 1 + _bar = 2 + begin + @bar = _foo + _bar + rescue Ractor::IsolationError + 42 + end + end + end + + def test + Foo.bar + end + + test + test + + Ractor.new { test }.value + "), @"42"); +} + +#[test] +fn test_global_tracepoint() { + assert_snapshot!(inspect(" + def foo = 1 + + foo + foo + + called = false + + tp = TracePoint.new(:return) { |event| + if event.method_id == :foo + called = true + end + } + tp.enable do + foo + end + called + "), @"true"); +} + +#[test] +fn test_local_tracepoint() { + assert_snapshot!(inspect(" + def foo = 1 + + foo + foo + + called = false + + tp = TracePoint.new(:return) { |_| called = true } + tp.enable(target: method(:foo)) do + foo + end + called + "), @"true"); +} + +// Regression test: TracePoint return value for methods with rescue that use `return`. +// ZJIT's send fallback uses rb_vm_opt_send_without_block which calls VM_EXEC, +// setting FLAG_FINISH on the callee frame. This changes how throw TAG_RETURN is +// handled, causing the return value to be nil instead of the actual value. +#[test] +fn test_tracepoint_return_value_with_rescue() { + assert_snapshot!(inspect(" + def f_raise + raise + rescue + return :f_raise_return + end + + ary = [] + TracePoint.new(:return, :b_return){|tp| + ary << [tp.event, tp.method_id, tp.return_value] + }.enable{ + send :f_raise + } + ary.pop # last b_return event is not required + ary + "), @"[[:return, :f_raise, :f_raise_return]]"); +} + +// Regression test: polymorphic getivar must not return nil for too-complex shapes. +// Too-complex shapes use hash tables for ivar storage, and rb_shape_get_iv_index() +// doesn't work for them. The polymorphic path must fall through to GetIvar instead. +#[test] +fn test_polymorphic_getivar_complex_shape() { + // Need threshold >= 3 so both shapes get profiled before compilation + set_call_threshold(3); + assert_snapshot!(inspect(r#" + class C + def initialize(foo) + @foo = foo + end + def foo = @foo + end + + # Create a normal object and a too-complex object of the same class + normal = C.new(:normal) + complex = C.new(:complex) + 1001.times { |i| complex.instance_variable_set(:"@v#{i}", i) } + 1001.times { |i| complex.remove_instance_variable(:"@v#{i}") } + + # Profile with both shapes before compilation triggers at call 3 + normal.foo # call 1: profile normal shape + complex.foo # call 2: profile too-complex shape + + # The too-complex object should still return :complex, not nil + [normal.foo, complex.foo] + "#), @"[:normal, :complex]"); +} + +/// When a method with keyword defaults contains a block that creates a lambda, +/// the lambda causes EP escape, which globally patches NoEPEscape PatchPoints. +/// On subsequent calls the PatchPoint side exit (which uses without_locals()) +/// must not leave stale keyword default values in the frame. We solve this by +/// invalidating the ISEQ version on EP escape so the interpreter takes over. +#[test] +fn test_ep_escape_preserves_keyword_default() { + set_call_threshold(1); + assert_snapshot!(inspect(r#" + def target(dumped, additional_methods: []) + dumped.class + additional_methods.each { |m| ->{ m } } + additional_methods + end + + def forwarder(x, **kwargs) + target(x, **kwargs) + end + + 5.times { forwarder("z") } + forwarder("y", additional_methods: [:to_s]) + target("x") + "#), @"[]"); +} + +#[test] +fn test_send_block_to_accepts_no_block() { + // Methods with &nil should raise ArgumentError when called with a block + assert_snapshot!(inspect(" + def m(a, &nil); a end + + def test + m(1) {} + rescue ArgumentError => e + e.message + end + + test + test + "), @r#""no block accepted""#); +} + +#[test] +fn test_send_block_to_method_not_using_block() { + // Passing a block to a method that doesn't use it should still work correctly. + // ZJIT falls back to the interpreter for this case so that unused block + // warnings are properly emitted. + assert_snapshot!(inspect(" + def m_no_block = 42 + + def test + m_no_block {} + end + + test + test + "), @"42"); +} + +#[test] +fn test_send_block_unused_warning_emitted_from_jit() { + // When ZJIT compiles a send with a block as a dynamic dispatch fallback + // (gen_send -> rb_vm_send), warn_unused_block uses cfp->pc for the dedup + // key. We save cfp->pc before calling rb_vm_send so the key is stable + // and won't spuriously collide with prior entries in the dedup table. + assert_snapshot!(inspect(r#" + $warnings = [] + module Warning + def warn(message, category: nil) + $warnings << message + end + end + + def m_unused_block_warn_test = 42 + + def test + $VERBOSE = true + m_unused_block_warn_test {} + $warnings.any? { |w| w.include?("may be ignored") } + end + + test + test + "#), @"true"); +} + +#[test] +fn test_load_immediates_into_registers_before_masking() { + // See https://github.com/ruby/ruby/pull/16669 -- this is a reduced reproduction from a Ruby + // spec. + set_call_threshold(2); + assert_snapshot!(inspect(r#" + def test + klass = Class.new do + def ===(o) + true + end + end + + case 1 + when klass.new + :called + end == :called + end + + test + test + "#), @"true"); +} + +#[test] +fn test_loop_terminates() { + set_call_threshold(3); + // Previous worklist-based type inference only worked for maximal SSA. This is a regression + // test for hanging. + assert_snapshot!(inspect(r#" + class TheClass + def set_value_loop + i = 0 + while i < 10 + @levar ||= i + i += 1 + end + end + end + + 3.times do |i| + TheClass.new.set_value_loop + end + "#), @"3"); +} + +// Regression test: getlocal with level=0 after setlocal_WC_0 was loading stale EP +// memory, causing Array#pack with buffer: keyword to receive the wrong buffer VALUE. +// See https://github.com/ruby/ruby/pull/16736 +#[test] +fn test_getlocal_level_zero_after_setlocal_wc_0() { + assert_snapshot!(inspect(r#" + def test + b = +"x" + v = 2 + [v].pack("C*", buffer: b) + b.size + end + test + "#), @"2"); +} diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 3ec2954c73..61de3709cb 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -81,6 +81,7 @@ #![allow(non_camel_case_types)] // A lot of imported CRuby globals aren't all-caps #![allow(non_upper_case_globals)] +#![allow(clippy::upper_case_acronyms)] // Some of this code may not be used yet #![allow(dead_code)] @@ -88,11 +89,13 @@ #![allow(unused_imports)] use std::convert::From; -use std::ffi::{CString, CStr}; -use std::fmt::{Debug, Formatter}; -use std::os::raw::{c_char, c_int, c_uint}; +use std::ffi::{c_void, CString, CStr}; +use std::fmt::{Debug, Display, Formatter}; +use std::os::raw::{c_char, c_int, c_long, c_uint}; use std::panic::{catch_unwind, UnwindSafe}; +use crate::cast::IntoUsize as _; + // We check that we can do this with the configure script and a couple of // static asserts. u64 and not usize to play nice with lowering to x86. pub type size_t = u64; @@ -129,9 +132,12 @@ unsafe extern "C" { pub fn rb_float_new(d: f64) -> VALUE; pub fn rb_hash_empty_p(hash: VALUE) -> VALUE; - pub fn rb_yjit_str_concat_codepoint(str: VALUE, codepoint: VALUE); + pub fn rb_ary_new_from_args(n: c_long, ...) -> VALUE; pub fn rb_str_setbyte(str: VALUE, index: VALUE, value: VALUE) -> VALUE; + pub fn rb_str_getbyte(str: VALUE, index: VALUE) -> VALUE; pub fn rb_vm_splat_array(flag: VALUE, ary: VALUE) -> VALUE; + pub fn rb_jit_fix_div_fix(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_jit_fix_mod_fix(x: VALUE, y: VALUE) -> VALUE; pub fn rb_vm_concat_array(ary1: VALUE, ary2st: VALUE) -> VALUE; pub fn rb_vm_get_special_object(reg_ep: *const VALUE, value_type: vm_special_object_type) -> VALUE; pub fn rb_vm_concat_to_array(ary1: VALUE, ary2st: VALUE) -> VALUE; @@ -144,6 +150,7 @@ unsafe extern "C" { ) -> bool; pub fn rb_vm_set_ivar_id(obj: VALUE, idx: u32, val: VALUE) -> VALUE; pub fn rb_vm_setinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, val: VALUE, ic: IVC); + pub fn rb_vm_getinstancevariable(iseq: IseqPtr, obj: VALUE, id: ID, ic: IVC) -> VALUE; pub fn rb_aliased_callable_method_entry( me: *const rb_callable_method_entry_t, ) -> *const rb_callable_method_entry_t; @@ -159,7 +166,13 @@ unsafe extern "C" { pub fn rb_vm_stack_canary() -> VALUE; pub fn rb_vm_push_cfunc_frame(cme: *const rb_callable_method_entry_t, recv_idx: c_int); pub fn rb_obj_class(klass: VALUE) -> VALUE; - pub fn rb_vm_objtostring(iseq: IseqPtr, recv: VALUE, cd: *const rb_call_data) -> VALUE; + pub fn rb_define_method( + klass: VALUE, + mid: *const c_char, + func: Option<unsafe extern "C" fn(args: VALUE) -> VALUE>, + arity: c_int, + ); + pub fn rb_vm_objtostring(reg_cfp: CfpPtr, recv: VALUE, cd: *const rb_call_data) -> VALUE; } // Renames @@ -187,21 +200,7 @@ pub use rb_get_iseq_body_local_iseq as get_iseq_body_local_iseq; pub use rb_get_iseq_body_iseq_encoded as get_iseq_body_iseq_encoded; pub use rb_get_iseq_body_stack_max as get_iseq_body_stack_max; pub use rb_get_iseq_body_type as get_iseq_body_type; -pub use rb_get_iseq_flags_has_lead as get_iseq_flags_has_lead; -pub use rb_get_iseq_flags_has_opt as get_iseq_flags_has_opt; -pub use rb_get_iseq_flags_has_kw as get_iseq_flags_has_kw; -pub use rb_get_iseq_flags_has_rest as get_iseq_flags_has_rest; -pub use rb_get_iseq_flags_has_post as get_iseq_flags_has_post; -pub use rb_get_iseq_flags_has_kwrest as get_iseq_flags_has_kwrest; -pub use rb_get_iseq_flags_has_block as get_iseq_flags_has_block; -pub use rb_get_iseq_flags_ambiguous_param0 as get_iseq_flags_ambiguous_param0; -pub use rb_get_iseq_flags_accepts_no_kwarg as get_iseq_flags_accepts_no_kwarg; pub use rb_get_iseq_body_local_table_size as get_iseq_body_local_table_size; -pub use rb_get_iseq_body_param_keyword as get_iseq_body_param_keyword; -pub use rb_get_iseq_body_param_size as get_iseq_body_param_size; -pub use rb_get_iseq_body_param_lead_num as get_iseq_body_param_lead_num; -pub use rb_get_iseq_body_param_opt_num as get_iseq_body_param_opt_num; -pub use rb_get_iseq_body_param_opt_table as get_iseq_body_param_opt_table; pub use rb_get_cikw_keyword_len as get_cikw_keyword_len; pub use rb_get_cikw_keywords_idx as get_cikw_keywords_idx; pub use rb_get_call_data_ci as get_call_data_ci; @@ -209,7 +208,6 @@ pub use rb_FL_TEST as FL_TEST; pub use rb_FL_TEST_RAW as FL_TEST_RAW; pub use rb_RB_TYPE_P as RB_TYPE_P; pub use rb_BASIC_OP_UNREDEFINED_P as BASIC_OP_UNREDEFINED_P; -pub use rb_RSTRUCT_LEN as RSTRUCT_LEN; pub use rb_vm_ci_argc as vm_ci_argc; pub use rb_vm_ci_mid as vm_ci_mid; pub use rb_vm_ci_flag as vm_ci_flag; @@ -217,6 +215,8 @@ pub use rb_vm_ci_kwarg as vm_ci_kwarg; pub use rb_METHOD_ENTRY_VISI as METHOD_ENTRY_VISI; pub use rb_RCLASS_ORIGIN as RCLASS_ORIGIN; pub use rb_vm_get_special_object as vm_get_special_object; +pub use rb_jit_fix_div_fix as rb_fix_div_fix; +pub use rb_jit_fix_mod_fix as rb_fix_mod_fix; /// Helper so we can get a Rust string for insn_name() pub fn insn_name(opcode: usize) -> String { @@ -241,10 +241,12 @@ pub fn insn_len(opcode: usize) -> u32 { } } -/// Opaque iseq type for opaque iseq pointers from vm_core.h +/// We avoid using bindgen for `rb_iseq_constant_body` since its definition changes depending +/// on build configuration while we need one bindgen file that works for all configurations. +/// Use an opaque type for it instead. /// See: <https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs> #[repr(C)] -pub struct rb_iseq_t { +pub struct rb_iseq_constant_body { _data: [u8; 0], _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, } @@ -253,7 +255,7 @@ pub struct rb_iseq_t { /// that this is a handle. Sometimes the C code briefly uses VALUE as /// an unsigned integer type and don't necessarily store valid handles but /// thankfully those cases are rare and don't cross the FFI boundary. -#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)] +#[derive(Copy, Clone, PartialEq, Eq, Hash)] #[repr(transparent)] // same size and alignment as simply `usize` pub struct VALUE(pub usize); @@ -266,10 +268,27 @@ pub struct ID(pub ::std::os::raw::c_ulong); /// Pointer to an ISEQ pub type IseqPtr = *const rb_iseq_t; +/// Index of a YARV instruction within an ISEQ's bytecode array. +pub type YarvInsnIdx = usize; + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct ShapeId(pub u32); -pub const INVALID_SHAPE_ID: ShapeId = ShapeId(RB_INVALID_SHAPE_ID); +pub const INVALID_SHAPE_ID: ShapeId = ShapeId(rb_invalid_shape_id); + +impl ShapeId { + pub fn is_valid(self) -> bool { + self != INVALID_SHAPE_ID + } + + pub fn is_complex(self) -> bool { + unsafe { rb_jit_shape_complex_p(self.0) } + } + + pub fn is_frozen(self) -> bool { + (self.0 & SHAPE_ID_FL_FROZEN) != 0 + } +} // Given an ISEQ pointer, convert PC to insn_idx pub fn iseq_pc_to_insn_idx(iseq: IseqPtr, pc: *mut VALUE) -> Option<u16> { @@ -283,6 +302,42 @@ pub fn iseq_opcode_at_idx(iseq: IseqPtr, insn_idx: u32) -> u32 { unsafe { rb_iseq_opcode_at_pc(iseq, pc) as u32 } } +/// Return true if a given ISEQ is known to escape EP to the heap on entry. +/// +/// As of vm_push_frame(), EP is always equal to BP. However, after pushing +/// a frame, some ISEQ setups call vm_bind_update_env(), which redirects EP. +pub fn iseq_escapes_ep(iseq: IseqPtr) -> bool { + match unsafe { get_iseq_body_type(iseq) } { + // The EP of the <main> frame points to TOPLEVEL_BINDING + ISEQ_TYPE_MAIN | + // eval frames point to the EP of another frame or scope + ISEQ_TYPE_EVAL => true, + _ => false, + } +} + +/// Index of the local variable that has a rest parameter if any +pub fn iseq_rest_param_idx(params: &IseqParameters) -> Option<i32> { + // TODO(alan): replace with `params.rest_start` + if params.flags.has_rest() != 0 { + Some(params.opt_num + params.lead_num) + } else { + None + } +} + +/// Iterate over all existing ISEQs +pub fn for_each_iseq<F: FnMut(IseqPtr)>(mut callback: F) { + unsafe extern "C" fn callback_wrapper(iseq: IseqPtr, data: *mut c_void) { + // SAFETY: points to the local below + let callback: *mut *mut dyn FnMut(IseqPtr) -> bool = data.cast(); + unsafe { (**callback)(iseq) }; + } + let mut data: *mut dyn FnMut(IseqPtr) = &raw mut callback; + let data: *mut *mut dyn FnMut(IseqPtr) = &raw mut data; + unsafe { rb_jit_for_each_iseq(Some(callback_wrapper), data.cast()) }; +} + /// Return a poison value to be set above the stack top to verify leafness. #[cfg(not(test))] pub fn vm_stack_canary() -> u64 { @@ -329,13 +384,6 @@ pub struct rb_callcache { _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, } -/// Opaque control_frame (CFP) struct from vm_core.h -#[repr(C)] -pub struct rb_control_frame_struct { - _data: [u8; 0], - _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, -} - /// Pointer to a control frame pointer (CFP) pub type CfpPtr = *mut rb_control_frame_struct; @@ -353,10 +401,27 @@ pub enum ClassRelationship { NoRelation, } +/// A print adapator for debug info about a [VALUE]. Includes info +/// the GC knows about the handle. Example: `println!("{}", value.obj_info());`. +pub struct ObjInfoPrinter(VALUE); + +impl Display for ObjInfoPrinter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + use std::mem::MaybeUninit; + const BUFFER_SIZE: usize = 0x100; + let mut buffer: MaybeUninit<[c_char; BUFFER_SIZE]> = MaybeUninit::uninit(); + let info = unsafe { + rb_raw_obj_info(buffer.as_mut_ptr().cast(), BUFFER_SIZE, self.0); + CStr::from_ptr(buffer.as_ptr().cast()).to_string_lossy() + }; + write!(f, "{info}") + } +} + impl VALUE { - /// Dump info about the value to the console similarly to rp(VALUE) - pub fn dump_info(self) { - unsafe { rb_obj_info_dump(self) } + /// Get a printer for raw debug info from `rb_obj_info()` about the value. + pub fn obj_info(self) -> ObjInfoPrinter { + ObjInfoPrinter(self) } /// Return whether the value is truthy or falsy in Ruby -- only nil and false are falsy. @@ -383,6 +448,11 @@ impl VALUE { !self.special_const_p() } + /// Shareability between ractors. `RB_OBJ_SHAREABLE_P()`. + pub fn shareable_p(self) -> bool { + (self.builtin_flags() & RUBY_FL_SHAREABLE as usize) != 0 + } + /// Return true if the value is a Ruby Fixnum (immediate-size integer) pub fn fixnum_p(self) -> bool { let VALUE(cval) = self; @@ -403,6 +473,34 @@ impl VALUE { self.static_sym_p() || self.dynamic_sym_p() } + pub fn instance_can_have_singleton_class(self) -> bool { + if self == unsafe { rb_cInteger } || self == unsafe { rb_cFloat } || + self == unsafe { rb_cSymbol } || self == unsafe { rb_cNilClass } || + self == unsafe { rb_cTrueClass } || self == unsafe { rb_cFalseClass } { + + return false + } + true + } + + /// A metaclass is the singleton class of an object that is a `Module`. + /// This is internal terminology from `class.c`. + pub fn is_metaclass(self) -> bool { + unsafe { + if RB_TYPE_P(self, RUBY_T_CLASS) && rb_zjit_singleton_class_p(self) { + let attached = rb_class_attached_object(self); + RB_TYPE_P(attached, RUBY_T_CLASS) || RB_TYPE_P(attached, RUBY_T_MODULE) + } else { + false + } + } + } + + pub fn is_singleton_class(self) -> bool { + // TODO(alan): clean up one of double check on T_CLASS + unsafe { RB_TYPE_P(self, RUBY_T_CLASS) && rb_zjit_singleton_class_p(self) } + } + /// Return true for a static (non-heap) Ruby symbol (RB_STATIC_SYM_P) pub fn static_sym_p(self) -> bool { let VALUE(cval) = self; @@ -450,8 +548,11 @@ impl VALUE { pub fn class_of(self) -> VALUE { if !self.special_const_p() { let builtin_type = self.builtin_type(); - assert_ne!(builtin_type, RUBY_T_NONE, "ZJIT should only see live objects"); - assert_ne!(builtin_type, RUBY_T_MOVED, "ZJIT should only see live objects"); + assert!( + builtin_type != RUBY_T_NONE && builtin_type != RUBY_T_MOVED, + "ZJIT saw a dead object. T_type={builtin_type}, {}", + self.obj_info() + ); } unsafe { rb_yarv_class_of(self) } @@ -489,20 +590,40 @@ impl VALUE { unsafe { rb_obj_frozen_p(self) != VALUE(0) } } - pub fn shape_too_complex(self) -> bool { - unsafe { rb_zjit_shape_obj_too_complex_p(self) } - } - pub fn shape_id_of(self) -> ShapeId { - ShapeId(unsafe { rb_obj_shape_id(self) }) + if self.special_const_p() { + INVALID_SHAPE_ID + } else { + ShapeId(unsafe { rb_obj_shape_id(self) }) + } } pub fn embedded_p(self) -> bool { unsafe { - FL_TEST_RAW(self, VALUE(ROBJECT_EMBED as usize)) != VALUE(0) + FL_TEST_RAW(self, VALUE(ROBJECT_HEAP as usize)) == VALUE(0) } } + pub fn struct_embedded_p(self) -> bool { + unsafe { + RB_TYPE_P(self, RUBY_T_STRUCT) && + FL_TEST_RAW(self, VALUE(RSTRUCT_EMBED_LEN_MASK)) != VALUE(0) + } + } + + pub fn class_fields_embedded_p(self) -> bool { + unsafe { rb_jit_class_fields_embedded_p(self) } + } + + pub fn data_p(self) -> bool { + !self.special_const_p() && + self.builtin_type() == RUBY_T_DATA + } + + pub fn data_fields_embedded_p(self) -> bool { + unsafe { rb_jit_data_fields_embedded_p(self) } + } + pub fn as_fixnum(self) -> i64 { assert!(self.fixnum_p()); (self.0 as i64) >> 1 @@ -602,6 +723,58 @@ impl VALUE { let k: isize = item.wrapping_add(item.wrapping_add(1)); VALUE(k as usize) } + + /// Call the write barrier after separately writing val to self. + pub fn write_barrier(self, val: VALUE) { + // rb_gc_writebarrier() asserts it is not called with a special constant + if !val.special_const_p() { + unsafe { rb_gc_writebarrier(self, val) }; + } + } +} + +pub type IseqParameters = rb_iseq_constant_body_rb_iseq_parameters; + +/// Extension trait to enable method calls on [`IseqPtr`] +pub trait IseqAccess { + unsafe fn params<'a>(self) -> &'a IseqParameters; +} + +impl IseqAccess for IseqPtr { + /// Get a description of the ISEQ's signature. Analogous to `ISEQ_BODY(iseq)->param` in C. + unsafe fn params<'a>(self) -> &'a IseqParameters { + use crate::cast::IntoUsize; + unsafe { &*((*self).body.byte_add(ISEQ_BODY_OFFSET_PARAM.to_usize()) as *const IseqParameters) } + } +} + +impl IseqParameters { + /// The `opt_table` is a mapping where `opt_table[number_of_optional_parameters_filled]` + /// gives the YARV entry point of ISeq as an index of the iseq_encoded array. + /// This method gives over the table that additionally works when `opt_num==0`, + /// when the table is stored as `NULL` and implicit. + /// The table stores the indexes as raw VALUE integers; they are not tagged as fixnum. + pub fn opt_table_slice(&self) -> &[VALUE] { + let opt_num: usize = self.opt_num.try_into().expect("ISeq opt_num should always >=0"); + if opt_num > 0 { + // The table has size=opt_num+1 because opt_table[opt_num] is valid (all optionals filled) + unsafe { std::slice::from_raw_parts(self.opt_table, opt_num + 1) } + } else { + // The ISeq entry point is index 0 when there are no optional parameters + &[VALUE(0)] + } + } +} + +impl Debug for VALUE { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Only use rb_obj_info() when {:#?} since it dereferences the pointer so carries some risk. + if f.alternate() { + write!(f, "VALUE({})", self.obj_info()) + } else { + write!(f, "VALUE(0x{:x})", self.0) + } + } } impl From<IseqPtr> for VALUE { @@ -682,21 +855,33 @@ impl ID { } } +impl Display for ID { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.contents_lossy()) + } +} + /// Produce a Ruby string from a Rust string slice pub fn rust_str_to_ruby(str: &str) -> VALUE { unsafe { rb_utf8_str_new(str.as_ptr() as *const _, str.len() as i64) } } -/// Produce a Ruby symbol from a Rust string slice -pub fn rust_str_to_sym(str: &str) -> VALUE { +/// Produce a Ruby ID from a Rust string slice +pub fn rust_str_to_id(str: &str) -> ID { let c_str = CString::new(str).unwrap(); let c_ptr: *const c_char = c_str.as_ptr(); - unsafe { rb_id2sym(rb_intern(c_ptr)) } + unsafe { rb_intern(c_ptr) } +} + +/// Produce a Ruby symbol from a Rust string slice +pub fn rust_str_to_sym(str: &str) -> VALUE { + let id = rust_str_to_id(str); + unsafe { rb_id2sym(id) } } /// Produce an owned Rust String from a C char pointer pub fn cstr_to_rust_string(c_char_ptr: *const c_char) -> Option<String> { - assert!(c_char_ptr != std::ptr::null()); + assert!(!c_char_ptr.is_null()); let c_str: &CStr = unsafe { CStr::from_ptr(c_char_ptr) }; @@ -718,6 +903,18 @@ pub fn iseq_name(iseq: IseqPtr) -> String { } } +// Equivalent of get_lvar_level() in compile.c +pub fn get_lvar_level(mut iseq: IseqPtr) -> u32 { + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; + let mut level = 0; + while iseq != local_iseq { + iseq = unsafe { rb_get_iseq_body_parent_iseq(iseq) }; + level += 1; + } + + level +} + // Location is the file defining the method, colon, method name. // Filenames are sometimes internal strings supplied to eval, // so be careful with them. @@ -726,29 +923,29 @@ pub fn iseq_get_location(iseq: IseqPtr, pos: u32) -> String { let iseq_lineno = unsafe { rb_iseq_line_no(iseq, pos as usize) }; let mut s = iseq_name(iseq); - s.push_str("@"); + s.push('@'); if iseq_path == Qnil { s.push_str("None"); } else { s.push_str(&ruby_str_to_rust_string(iseq_path)); } - s.push_str(":"); + s.push(':'); s.push_str(&iseq_lineno.to_string()); s } +pub fn ruby_str_to_rust_string_result(v: VALUE) -> Result<String, std::string::FromUtf8Error> { + let str_ptr = unsafe { rb_RSTRING_PTR(v) } as *mut u8; + let str_len: usize = unsafe { rb_RSTRING_LEN(v) }.try_into().unwrap(); + let str_slice: &[u8] = unsafe { std::slice::from_raw_parts(str_ptr, str_len) }; + String::from_utf8(str_slice.to_vec()) +} // Convert a CRuby UTF-8-encoded RSTRING into a Rust string. // This should work fine on ASCII strings and anything else // that is considered legal UTF-8, including embedded nulls. -fn ruby_str_to_rust_string(v: VALUE) -> String { - let str_ptr = unsafe { rb_RSTRING_PTR(v) } as *mut u8; - let str_len: usize = unsafe { rb_RSTRING_LEN(v) }.try_into().unwrap(); - let str_slice: &[u8] = unsafe { std::slice::from_raw_parts(str_ptr, str_len) }; - match String::from_utf8(str_slice.to_vec()) { - Ok(utf8) => utf8, - Err(_) => String::new(), - } +pub fn ruby_str_to_rust_string(v: VALUE) -> String { + ruby_str_to_rust_string_result(v).unwrap_or_default() } pub fn ruby_sym_to_rust_string(v: VALUE) -> String { @@ -819,7 +1016,15 @@ where let line = loc.line; let mut recursive_lock_level: c_uint = 0; - unsafe { rb_zjit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; + unsafe { rb_jit_vm_lock_then_barrier(&mut recursive_lock_level, file, line) }; + // Ensure GC is off while we have the VM lock because: + // 1. We create many transient Rust collections that hold VALUEs during compilation. + // It's extremely tricky to properly marked and reference update these, not to + // mention the overhead and ergonomics issues. + // 2. If we yield to the GC while compiling, it re-enters our mark and update functions. + // This breaks `&mut` exclusivity since mark functions derive fresh `&mut` from statics + // while there is a stack frame below it that has an overlapping `&mut`. That's UB. + let gc_disabled_pre_call = unsafe { rb_gc_disable() }.test(); let ret = match catch_unwind(func) { Ok(result) => result, @@ -839,7 +1044,12 @@ where } }; - unsafe { rb_zjit_vm_unlock(&mut recursive_lock_level, file, line) }; + unsafe { + if !gc_disabled_pre_call { + rb_gc_enable(); + } + rb_jit_vm_unlock(&mut recursive_lock_level, file, line); + }; ret } @@ -876,7 +1086,7 @@ pub fn rb_bug_panic_hook() { // You may also use ZJIT_RB_BUG=1 to trigger this on dev builds. if release_build || env::var("ZJIT_RB_BUG").is_ok() { // Abort with rb_bug(). It has a length limit on the message. - let panic_message = &format!("{}", panic_info)[..]; + let panic_message = &format!("{panic_info}")[..]; let len = std::cmp::min(0x100, panic_message.len()) as c_int; unsafe { rb_bug(b"ZJIT: %*s\0".as_ref().as_ptr() as *const c_char, len, panic_message.as_ptr()); } } else { @@ -900,8 +1110,9 @@ mod manual_defs { use super::*; pub const SIZEOF_VALUE: usize = 8; + pub const BITS_PER_BYTE: usize = 8; pub const SIZEOF_VALUE_I32: i32 = SIZEOF_VALUE as i32; - pub const VALUE_BITS: u8 = 8 * SIZEOF_VALUE as u8; + pub const VALUE_BITS: u8 = BITS_PER_BYTE as u8 * SIZEOF_VALUE as u8; pub const RUBY_LONG_MIN: isize = std::os::raw::c_long::MIN as isize; pub const RUBY_LONG_MAX: isize = std::os::raw::c_long::MAX as isize; @@ -954,12 +1165,6 @@ mod manual_defs { pub const RUBY_OFFSET_CFP_JIT_RETURN: i32 = 48; pub const RUBY_SIZEOF_CONTROL_FRAME: usize = 56; - // Constants from rb_execution_context_t vm_core.h - pub const RUBY_OFFSET_EC_CFP: i32 = 16; - pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: i32 = 32; // rb_atomic_t (u32) - pub const RUBY_OFFSET_EC_INTERRUPT_MASK: i32 = 36; // rb_atomic_t (u32) - pub const RUBY_OFFSET_EC_THREAD_PTR: i32 = 48; - // Constants from rb_thread_t in vm_core.h pub const RUBY_OFFSET_THREAD_SELF: i32 = 16; @@ -973,7 +1178,7 @@ pub use manual_defs::*; pub mod test_utils { use std::{ptr::null, sync::Once}; - use crate::{options::rb_zjit_prepare_options, state::rb_zjit_enabled_p, state::ZJITState}; + use crate::{options::{rb_zjit_call_threshold, rb_zjit_prepare_options, set_call_threshold, DEFAULT_CALL_THRESHOLD}, state::{rb_zjit_entry, ZJITState}}; use super::*; @@ -998,6 +1203,11 @@ pub mod test_utils { rb_zjit_prepare_options(); // enable `#with_jit` on builtins ruby_init(); + // The default rb_zjit_profile_threshold is too high, so lower it for HIR tests. + if rb_zjit_call_threshold == DEFAULT_CALL_THRESHOLD { + set_call_threshold(2); + } + // Pass command line options so the VM loads core library methods defined in // ruby such as from `kernel.rb`. // We drive ZJIT manually in tests, so disable heuristic compilation triggers. @@ -1010,35 +1220,41 @@ pub mod test_utils { crate::cruby::ids::init(); // for ID! usages in tests } + // Call ZJIT hooks to install Ruby implementations of builtins like Array#each + unsafe { + let zjit = rb_const_get(rb_cRubyVM, rust_str_to_id("ZJIT")); + rb_funcallv(zjit, rust_str_to_id("call_jit_hooks"), 0, std::ptr::null()); + } + // Set up globals for convenience - ZJITState::init(); + let zjit_entry = ZJITState::init(); // Enable zjit_* instructions - unsafe { rb_zjit_enabled_p = true; } + unsafe { rb_zjit_entry = zjit_entry; } } /// Make sure the Ruby VM is set up and run a given callback with rb_protect() pub fn with_rubyvm<T>(mut func: impl FnMut() -> T) -> T { - RUBY_VM_INIT.call_once(|| boot_rubyvm()); - - // Set up a callback wrapper to store a return value - let mut result: Option<T> = None; - let mut data: &mut dyn FnMut() = &mut || { - // Store the result externally - result.replace(func()); - }; + RUBY_VM_INIT.call_once(boot_rubyvm); // Invoke callback through rb_protect() so exceptions don't crash the process. // "Fun" double pointer dance to get a thin function pointer to pass through C unsafe extern "C" fn callback_wrapper(data: VALUE) -> VALUE { // SAFETY: shorter lifetime than the data local in the caller frame - let callback: &mut &mut dyn FnMut() = unsafe { std::mem::transmute(data) }; - callback(); + let callback: *const *mut dyn FnMut() = std::ptr::with_exposed_provenance_mut(data.0); + unsafe { (**callback)() }; Qnil } + // Set up a callback wrapper to store the return value + let mut result: Option<T> = None; + let mut func_wrapper = || { + result.replace(func()); + }; + let data: *mut dyn FnMut() = &raw mut func_wrapper; + let data: *const *mut dyn FnMut() = &raw const data; let mut state: c_int = 0; - unsafe { super::rb_protect(Some(callback_wrapper), VALUE((&mut data) as *mut _ as usize), &mut state) }; + unsafe { super::rb_protect(Some(callback_wrapper), VALUE(data.expose_provenance()), &mut state) }; if state != 0 { unsafe { rb_zjit_print_exception(); } assert_eq!(0, state, "Exceptional unwind in callback. Ruby exception?"); @@ -1068,9 +1284,55 @@ pub mod test_utils { }) } - /// Get the ISeq of a specified method + /// Evaluate a given Ruby program with compile options + pub fn eval_with_options(program: &str, options_expr: &str) -> VALUE { + with_rubyvm(|| { + let options = eval(options_expr); + let wrapped_iseq = compile_to_wrapped_iseq_with_options(&unindent(program, false), options); + unsafe { rb_funcallv(wrapped_iseq, ID!(eval), 0, null()) } + }) + } + + /// Get the #inspect of a given Ruby program in Rust string + pub fn inspect(program: &str) -> String { + let inspect = format!("({program}).inspect"); + ruby_str_to_rust_string(eval(&inspect)) + } + + /// Like inspect, but also asserts that all compilations triggered by this program succeed. + pub fn assert_compiles_allowing_exits(program: &str) -> String { + use crate::state::ZJITState; + ZJITState::enable_assert_compiles(); + let result = inspect(program); + ZJITState::disable_assert_compiles(); + result + } + + /// Like inspect, but also asserts that all compilations triggered by this program succeed and + /// no side exits occurr during the program. + pub fn assert_compiles(program: &str) -> String { + use crate::state::ZJITState; + let exits_before = crate::stats::total_exit_count(); + ZJITState::enable_assert_compiles(); + let result = inspect(program); + ZJITState::disable_assert_compiles(); + assert_eq!(exits_before, crate::stats::total_exit_count(), "Program side-exited"); + result + } + + /// Get IseqPtr for a specified method pub fn get_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { - let wrapped_iseq = eval(&format!("RubyVM::InstructionSequence.of({}.method(:{}))", recv, name)); + get_proc_iseq(&format!("{}.method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified instance method + pub fn get_instance_method_iseq(recv: &str, name: &str) -> *const rb_iseq_t { + get_proc_iseq(&format!("{}.instance_method(:{})", recv, name)) + } + + /// Get IseqPtr for a specified Proc object + pub fn get_proc_iseq(obj: &str) -> *const rb_iseq_t { + let wrapped_iseq = eval(&format!("RubyVM::InstructionSequence.of({obj})")); unsafe { rb_iseqw_to_iseq(wrapped_iseq) } } @@ -1099,7 +1361,7 @@ pub mod test_utils { if line.len() > spaces { unindented.extend_from_slice(&line.as_bytes()[spaces..]); } else { - unindented.extend_from_slice(&line.as_bytes()); + unindented.extend_from_slice(line.as_bytes()); } } String::from_utf8(unindented).unwrap() @@ -1107,10 +1369,15 @@ pub mod test_utils { /// Compile a program into a RubyVM::InstructionSequence object fn compile_to_wrapped_iseq(program: &str) -> VALUE { + compile_to_wrapped_iseq_with_options(program, Qnil) + } + + fn compile_to_wrapped_iseq_with_options(program: &str, options: VALUE) -> VALUE { let bytes = program.as_bytes().as_ptr() as *const c_char; unsafe { let program_str = rb_utf8_str_new(bytes, program.len().try_into().unwrap()); - rb_funcallv(rb_cISeq, ID!(compile), 1, &program_str) + let args = [program_str, Qnil, Qnil, VALUE(1_usize.wrapping_shl(1) | 1), options]; + rb_funcallv(rb_cISeq, ID!(compile), args.len() as c_int, args.as_ptr()) } } @@ -1171,20 +1438,152 @@ pub mod test_utils { fn value_from_fixnum_too_small_isize() { assert_eq!(VALUE::fixnum_from_isize(RUBY_FIXNUM_MIN-1), VALUE(1)); } + + #[test] + fn value_fmt_debug() { + assert_eq!("VALUE(0xcafe)", format!("{:?}", VALUE(0xcafe))); + let alternate = format!("{:#?}", eval("::Hash")); + assert!(alternate.contains("Hash"), "'Hash' not substring of '{alternate}'"); + } } #[cfg(test)] pub use test_utils::*; -/// Get class name from a class pointer. +/// Get class name from a class pointer. For anonymous classes, includes the +/// superclass name for context (e.g. `#<Class(String):0x00007f...>`). pub fn get_class_name(class: VALUE) -> String { // type checks for rb_class2name() - if unsafe { RB_TYPE_P(class, RUBY_T_MODULE) || RB_TYPE_P(class, RUBY_T_CLASS) } { + let name = if unsafe { RB_TYPE_P(class, RUBY_T_MODULE) || RB_TYPE_P(class, RUBY_T_CLASS) } { Some(class) } else { None }.and_then(|class| unsafe { cstr_to_rust_string(rb_class2name(class)) - }).unwrap_or_else(|| "Unknown".to_string()) + }).unwrap_or_else(|| "Unknown".to_string()); + + // For anonymous classes, include the superclass name for context. + // Use rb_class_real to resolve through iclasses (internal include/prepend + // wrappers) before checking rb_mod_name, which returns Qnil for anonymous classes. + // e.g. "#<Class:0x7f...>" with superclass String => "#<Class(String):0x7f...>" + if unsafe { RB_TYPE_P(class, RUBY_T_CLASS) && rb_mod_name(rb_class_real(class)) == Qnil } { + let super_class = unsafe { rb_class_get_superclass(class) }; + if super_class != Qnil { + let super_name = get_class_name(super_class); + return format!("#<Class({super_name}):{:#x}>", class.0); + } + } + + name +} + +// Return the module name for a given module or class. For anonymous modules, returns None since +// rb_mod_name returns Qnil. +pub fn get_module_name(module: VALUE) -> Option<String> { + // type checks for rb_mod_name() + assert!(unsafe { RB_TYPE_P(module, RUBY_T_MODULE) || RB_TYPE_P(module, RUBY_T_CLASS) }, "Expected class or module"); + let name = unsafe { rb_mod_name(module) }; + if name == Qnil { + None + } else { + Some(ruby_str_to_rust_string(name)) + } +} + + +#[cfg(test)] +mod class_name_tests { + use super::*; + use test_utils::{eval, with_rubyvm}; + + #[test] + fn named_class() { + with_rubyvm(|| { + assert_eq!(get_class_name(eval("String")), "String"); + }); + } + + #[test] + fn named_module() { + with_rubyvm(|| { + assert_eq!(get_class_name(eval("Kernel")), "Kernel"); + }); + } + + #[test] + fn anonymous_class_includes_superclass() { + with_rubyvm(|| { + let name = get_class_name(eval("Class.new(String)")); + assert!(name.starts_with("#<Class(String):0x"), "got: {name}"); + }); + } + + #[test] + fn anonymous_class_nested_superclass() { + with_rubyvm(|| { + let name = get_class_name(eval("Class.new(Class.new(String))")); + assert!(name.starts_with("#<Class(#<Class(String):0x"), "got: {name}"); + }); + } + + #[test] + fn anonymous_module_unchanged() { + with_rubyvm(|| { + let name = get_class_name(eval("Module.new")); + assert!(name.starts_with("#<Module:0x"), "got: {name}"); + }); + } + +} + +pub fn class_has_leaf_allocator(class: VALUE) -> bool { + // We need to check if the class is initialized and not a singleton before + // trying to read the allocator, otherwise it will raise. + // Because of this they should be considered non-leaf anyways. + if !unsafe { rb_zjit_class_initialized_p(class) } { return false; } + if unsafe { rb_zjit_singleton_class_p(class) } { return false; } + + // empty_hash_alloc + if class == unsafe { rb_cHash } { return true; } + // empty_ary_alloc + if class == unsafe { rb_cArray } { return true; } + // empty_str_alloc + if class == unsafe { rb_cString } { return true; } + // rb_reg_s_alloc + if class == unsafe { rb_cRegexp } { return true; } + // rb_class_allocate_instance + unsafe { rb_zjit_class_has_default_allocator(class) } +} + +/// Whether a method ISEQ defined on `owner` is guaranteed to run with a `self` +/// that is a heap (non-immediate) object. +/// +/// True only for plain `def` methods (`ISEQ_TYPE_METHOD`) defined on a normal, +/// initialized, non-singleton class that uses the default allocator +/// (`rb_class_allocate_instance`). The receiver of such a method is always +/// `kind_of?` the owner, and no user class with the default allocator can be +/// inserted into the ancestry of an immediate, so `self` cannot be an immediate. +/// +/// The default-allocator check alone is not sufficient: `Object`, `BasicObject`, +/// and `Numeric` use the default allocator yet are ancestors of immediates (e.g. +/// `Integer`). Every such class is also an ancestor of `Integer`, so a single +/// `rb_obj_is_kind_of(<a fixnum>, owner)` check rules all of them out. +/// +/// Returns `false` conservatively for anything that doesn't clearly qualify +/// (modules, singleton classes, custom allocators, non-`def` ISEQs, etc.). +pub fn iseq_self_is_heap_object(iseq: IseqPtr, owner: VALUE) -> bool { + if unsafe { rb_get_iseq_body_type(iseq) } != ISEQ_TYPE_METHOD { return false; } + if !unsafe { RB_TYPE_P(owner, RUBY_T_CLASS) } { return false; } + // Check initialized + non-singleton before reading the allocator (reading it otherwise + // aborts). + // TODO(max): Determine if we can loosen this to allow methods defined on singleton classes. + if !unsafe { rb_zjit_class_initialized_p(owner) } { return false; } + if unsafe { rb_zjit_singleton_class_p(owner) } { return false; } + if !unsafe { rb_zjit_class_has_default_allocator(owner) } { return false; } + // Exclude Object/BasicObject/Numeric and friends: classes that use the default + // allocator but sit above an immediate class in the ancestry chain. They are + // all ancestors of Integer, so this single check covers every immediate type. + if unsafe { rb_obj_is_kind_of(VALUE::fixnum_from_usize(0), owner) }.test() { return false; } + true } /// Interned ID values for Ruby symbols and method names. @@ -1222,6 +1621,7 @@ pub(crate) mod ids { name: NULL content: b"" name: respond_to_missing content: b"respond_to_missing?" name: eq content: b"==" + name: string_eq content: b"String#==" name: include_p content: b"include?" name: to_ary name: to_s @@ -1239,9 +1639,18 @@ pub(crate) mod ids { name: ge content: b">=" name: and content: b"&" name: or content: b"|" + name: xor content: b"^" name: freeze name: minusat content: b"-@" name: aref content: b"[]" + name: rb_ivar_get_at_no_ractor_check + name: RUBY_FL_FREEZE + name: RUBY_ELTS_SHARED + name: RubyVM + name: ZJIT + name: induce_side_exit_bang content: b"induce_side_exit!" + name: induce_compile_failure_bang content: b"induce_compile_failure!" + name: induce_breakpoint_bang content: b"induce_breakpoint!" } /// Get an CRuby `ID` to an interned string, e.g. a particular method name. @@ -1249,7 +1658,7 @@ pub(crate) mod ids { ($id_name:ident) => {{ let id = $crate::cruby::ids::$id_name.load(std::sync::atomic::Ordering::Relaxed); debug_assert_ne!(0, id, "ids module should be initialized"); - ID(id) + $crate::cruby::ID(id) }} } pub(crate) use ID; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index ee6d4d5e0e..cc6a37d827 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1,6 +1,143 @@ /* automatically generated by rust-bindgen 0.71.1 */ #[repr(C)] +#[derive(Copy, Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct __BindgenBitfieldUnit<Storage> { + storage: Storage, +} +impl<Storage> __BindgenBitfieldUnit<Storage> { + #[inline] + pub const fn new(storage: Storage) -> Self { + Self { storage } + } +} +impl<Storage> __BindgenBitfieldUnit<Storage> +where + Storage: AsRef<[u8]> + AsMut<[u8]>, +{ + #[inline] + fn extract_bit(byte: u8, index: usize) -> bool { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + byte & mask == mask + } + #[inline] + pub fn get_bit(&self, index: usize) -> bool { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = self.storage.as_ref()[byte_index]; + Self::extract_bit(byte, index) + } + #[inline] + pub unsafe fn raw_get_bit(this: *const Self, index: usize) -> bool { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = *(core::ptr::addr_of!((*this).storage) as *const u8).offset(byte_index as isize); + Self::extract_bit(byte, index) + } + #[inline] + fn change_bit(byte: u8, index: usize, val: bool) -> u8 { + let bit_index = if cfg!(target_endian = "big") { + 7 - (index % 8) + } else { + index % 8 + }; + let mask = 1 << bit_index; + if val { + byte | mask + } else { + byte & !mask + } + } + #[inline] + pub fn set_bit(&mut self, index: usize, val: bool) { + debug_assert!(index / 8 < self.storage.as_ref().len()); + let byte_index = index / 8; + let byte = &mut self.storage.as_mut()[byte_index]; + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub unsafe fn raw_set_bit(this: *mut Self, index: usize, val: bool) { + debug_assert!(index / 8 < core::mem::size_of::<Storage>()); + let byte_index = index / 8; + let byte = + (core::ptr::addr_of_mut!((*this).storage) as *mut u8).offset(byte_index as isize); + *byte = Self::change_bit(*byte, index, val); + } + #[inline] + pub fn get(&self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if self.get_bit(i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub unsafe fn raw_get(this: *const Self, bit_offset: usize, bit_width: u8) -> u64 { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + let mut val = 0; + for i in 0..(bit_width as usize) { + if Self::raw_get_bit(this, i + bit_offset) { + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + val |= 1 << index; + } + } + val + } + #[inline] + pub fn set(&mut self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < self.storage.as_ref().len()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= self.storage.as_ref().len()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + self.set_bit(index + bit_offset, val_bit_is_set); + } + } + #[inline] + pub unsafe fn raw_set(this: *mut Self, bit_offset: usize, bit_width: u8, val: u64) { + debug_assert!(bit_width <= 64); + debug_assert!(bit_offset / 8 < core::mem::size_of::<Storage>()); + debug_assert!((bit_offset + (bit_width as usize)) / 8 <= core::mem::size_of::<Storage>()); + for i in 0..(bit_width as usize) { + let mask = 1 << i; + let val_bit_is_set = val & mask == mask; + let index = if cfg!(target_endian = "big") { + bit_width as usize - 1 - i + } else { + i + }; + Self::raw_set_bit(this, index + bit_offset, val_bit_is_set); + } + } +} +#[repr(C)] #[derive(Default)] pub struct __IncompleteArrayField<T>(::std::marker::PhantomData<T>, [T; 0]); impl<T> __IncompleteArrayField<T> { @@ -30,6 +167,49 @@ impl<T> ::std::fmt::Debug for __IncompleteArrayField<T> { fmt.write_str("__IncompleteArrayField") } } +#[repr(C)] +pub struct __BindgenUnionField<T>(::std::marker::PhantomData<T>); +impl<T> __BindgenUnionField<T> { + #[inline] + pub const fn new() -> Self { + __BindgenUnionField(::std::marker::PhantomData) + } + #[inline] + pub unsafe fn as_ref(&self) -> &T { + ::std::mem::transmute(self) + } + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut T { + ::std::mem::transmute(self) + } +} +impl<T> ::std::default::Default for __BindgenUnionField<T> { + #[inline] + fn default() -> Self { + Self::new() + } +} +impl<T> ::std::clone::Clone for __BindgenUnionField<T> { + #[inline] + fn clone(&self) -> Self { + *self + } +} +impl<T> ::std::marker::Copy for __BindgenUnionField<T> {} +impl<T> ::std::fmt::Debug for __BindgenUnionField<T> { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + fmt.write_str("__BindgenUnionField") + } +} +impl<T> ::std::hash::Hash for __BindgenUnionField<T> { + fn hash<H: ::std::hash::Hasher>(&self, _state: &mut H) {} +} +impl<T> ::std::cmp::PartialEq for __BindgenUnionField<T> { + fn eq(&self, _other: &__BindgenUnionField<T>) -> bool { + true + } +} +impl<T> ::std::cmp::Eq for __BindgenUnionField<T> {} pub const ONIG_OPTION_IGNORECASE: u32 = 1; pub const ONIG_OPTION_EXTEND: u32 = 2; pub const ONIG_OPTION_MULTILINE: u32 = 4; @@ -47,12 +227,14 @@ pub const NIL_REDEFINED_OP_FLAG: u32 = 512; pub const TRUE_REDEFINED_OP_FLAG: u32 = 1024; pub const FALSE_REDEFINED_OP_FLAG: u32 = 2048; pub const PROC_REDEFINED_OP_FLAG: u32 = 4096; +pub const VM_KW_SPECIFIED_BITS_MAX: u32 = 31; pub const VM_ENV_DATA_SIZE: u32 = 3; pub const VM_ENV_DATA_INDEX_ME_CREF: i32 = -2; pub const VM_ENV_DATA_INDEX_SPECVAL: i32 = -1; pub const VM_ENV_DATA_INDEX_FLAGS: u32 = 0; pub const VM_BLOCK_HANDLER_NONE: u32 = 0; pub const SHAPE_ID_NUM_BITS: u32 = 32; +pub const ZJIT_JIT_RETURN_C_FRAME: u32 = 1; pub type rb_alloc_func_t = ::std::option::Option<unsafe extern "C" fn(klass: VALUE) -> VALUE>; pub const RUBY_Qfalse: ruby_special_consts = 0; pub const RUBY_Qnil: ruby_special_consts = 4; @@ -105,11 +287,9 @@ pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32; pub const RUBY_FL_PROMOTED: ruby_fl_type = 32; pub const RUBY_FL_UNUSED6: ruby_fl_type = 64; pub const RUBY_FL_FINALIZE: ruby_fl_type = 128; -pub const RUBY_FL_TAINT: ruby_fl_type = 0; pub const RUBY_FL_EXIVAR: ruby_fl_type = 0; pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256; -pub const RUBY_FL_UNTRUSTED: ruby_fl_type = 0; -pub const RUBY_FL_UNUSED9: ruby_fl_type = 512; +pub const RUBY_FL_WEAK_REFERENCE: ruby_fl_type = 512; pub const RUBY_FL_UNUSED10: ruby_fl_type = 1024; pub const RUBY_FL_FREEZE: ruby_fl_type = 2048; pub const RUBY_FL_USER0: ruby_fl_type = 4096; @@ -160,8 +340,25 @@ pub const RARRAY_EMBED_LEN_SHIFT: ruby_rarray_consts = 15; pub type ruby_rarray_consts = u32; pub const RMODULE_IS_REFINEMENT: ruby_rmodule_flags = 8192; pub type ruby_rmodule_flags = u32; -pub const ROBJECT_EMBED: ruby_robject_flags = 8192; +pub const ROBJECT_HEAP: ruby_robject_flags = 65536; pub type ruby_robject_flags = u32; +pub const RUBY_TYPED_FREE_IMMEDIATELY: rbimpl_typeddata_flags = 1; +pub const RUBY_TYPED_EMBEDDABLE: rbimpl_typeddata_flags = 2; +pub const RUBY_TYPED_FROZEN_SHAREABLE: rbimpl_typeddata_flags = 256; +pub const RUBY_TYPED_WB_PROTECTED: rbimpl_typeddata_flags = 32; +pub const RUBY_TYPED_DECL_MARKING: rbimpl_typeddata_flags = 16384; +pub type rbimpl_typeddata_flags = u32; +pub type rb_event_flag_t = u32; +pub type rb_block_call_func = ::std::option::Option< + unsafe extern "C" fn( + yielded_arg: VALUE, + callback_arg: VALUE, + argc: ::std::os::raw::c_int, + argv: *const VALUE, + blockarg: VALUE, + ) -> VALUE, +>; +pub type rb_block_call_func_t = rb_block_call_func; pub const RUBY_ENCODING_INLINE_MAX: ruby_encoding_consts = 127; pub const RUBY_ENCODING_SHIFT: ruby_encoding_consts = 22; pub const RUBY_ENCODING_MASK: ruby_encoding_consts = 532676608; @@ -200,24 +397,38 @@ pub const BOP_NIL_P: ruby_basic_operators = 15; pub const BOP_SUCC: ruby_basic_operators = 16; pub const BOP_GT: ruby_basic_operators = 17; pub const BOP_GE: ruby_basic_operators = 18; -pub const BOP_NOT: ruby_basic_operators = 19; -pub const BOP_NEQ: ruby_basic_operators = 20; -pub const BOP_MATCH: ruby_basic_operators = 21; -pub const BOP_FREEZE: ruby_basic_operators = 22; -pub const BOP_UMINUS: ruby_basic_operators = 23; -pub const BOP_MAX: ruby_basic_operators = 24; -pub const BOP_MIN: ruby_basic_operators = 25; -pub const BOP_HASH: ruby_basic_operators = 26; -pub const BOP_CALL: ruby_basic_operators = 27; -pub const BOP_AND: ruby_basic_operators = 28; -pub const BOP_OR: ruby_basic_operators = 29; -pub const BOP_CMP: ruby_basic_operators = 30; -pub const BOP_DEFAULT: ruby_basic_operators = 31; -pub const BOP_PACK: ruby_basic_operators = 32; -pub const BOP_INCLUDE_P: ruby_basic_operators = 33; -pub const BOP_LAST_: ruby_basic_operators = 34; +pub const BOP_GTGT: ruby_basic_operators = 19; +pub const BOP_NOT: ruby_basic_operators = 20; +pub const BOP_NEQ: ruby_basic_operators = 21; +pub const BOP_MATCH: ruby_basic_operators = 22; +pub const BOP_FREEZE: ruby_basic_operators = 23; +pub const BOP_UMINUS: ruby_basic_operators = 24; +pub const BOP_MAX: ruby_basic_operators = 25; +pub const BOP_MIN: ruby_basic_operators = 26; +pub const BOP_HASH: ruby_basic_operators = 27; +pub const BOP_CALL: ruby_basic_operators = 28; +pub const BOP_AND: ruby_basic_operators = 29; +pub const BOP_OR: ruby_basic_operators = 30; +pub const BOP_CMP: ruby_basic_operators = 31; +pub const BOP_DEFAULT: ruby_basic_operators = 32; +pub const BOP_PACK: ruby_basic_operators = 33; +pub const BOP_INCLUDE_P: ruby_basic_operators = 34; +pub const BOP_LAST_: ruby_basic_operators = 35; pub type ruby_basic_operators = u32; pub type rb_serial_t = ::std::os::raw::c_ulonglong; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_id_item { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_id_table { + pub capa: ::std::os::raw::c_int, + pub num: ::std::os::raw::c_int, + pub used: ::std::os::raw::c_int, + pub items: *mut rb_id_item, +} pub const imemo_env: imemo_type = 0; pub const imemo_cref: imemo_type = 1; pub const imemo_svar: imemo_type = 2; @@ -227,11 +438,29 @@ pub const imemo_memo: imemo_type = 5; pub const imemo_ment: imemo_type = 6; pub const imemo_iseq: imemo_type = 7; pub const imemo_tmpbuf: imemo_type = 8; +pub const imemo_cvar_entry: imemo_type = 9; pub const imemo_callinfo: imemo_type = 10; pub const imemo_callcache: imemo_type = 11; pub const imemo_constcache: imemo_type = 12; pub const imemo_fields: imemo_type = 13; +pub const imemo_subclasses: imemo_type = 14; +pub const imemo_cdhash: imemo_type = 15; pub type imemo_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct vm_ifunc_argc { + pub min: ::std::os::raw::c_int, + pub max: ::std::os::raw::c_int, +} +#[repr(C)] +pub struct vm_ifunc { + pub flags: VALUE, + pub svar_lep: *mut VALUE, + pub func: rb_block_call_func_t, + pub data: *const ::std::os::raw::c_void, + pub argc: vm_ifunc_argc, +} +pub type rb_atomic_t = ::std::os::raw::c_uint; pub const METHOD_VISI_UNDEF: rb_method_visibility_t = 0; pub const METHOD_VISI_PUBLIC: rb_method_visibility_t = 1; pub const METHOD_VISI_PRIVATE: rb_method_visibility_t = 2; @@ -269,6 +498,7 @@ pub const VM_METHOD_TYPE_OPTIMIZED: rb_method_type_t = 9; pub const VM_METHOD_TYPE_MISSING: rb_method_type_t = 10; pub const VM_METHOD_TYPE_REFINED: rb_method_type_t = 11; pub type rb_method_type_t = u32; +pub type rb_iseq_t = rb_iseq_struct; pub type rb_cfunc_t = ::std::option::Option<unsafe extern "C" fn() -> VALUE>; #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -291,7 +521,22 @@ pub const OPTIMIZED_METHOD_TYPE_STRUCT_AREF: method_optimized_type = 3; pub const OPTIMIZED_METHOD_TYPE_STRUCT_ASET: method_optimized_type = 4; pub const OPTIMIZED_METHOD_TYPE__MAX: method_optimized_type = 5; pub type method_optimized_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_position_struct { + pub lineno: ::std::os::raw::c_int, + pub column: ::std::os::raw::c_int, +} +pub type rb_code_position_t = rb_code_position_struct; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_code_location_struct { + pub beg_pos: rb_code_position_t, + pub end_pos: rb_code_position_t, +} +pub type rb_code_location_t = rb_code_location_struct; pub type rb_num_t = ::std::os::raw::c_ulong; +pub type rb_snum_t = ::std::os::raw::c_long; pub const RUBY_TAG_NONE: ruby_tag_type = 0; pub const RUBY_TAG_RETURN: ruby_tag_type = 1; pub const RUBY_TAG_BREAK: ruby_tag_type = 2; @@ -310,8 +555,6 @@ pub type ruby_vm_throw_flags = u32; pub struct iseq_inline_constant_cache_entry { pub flags: VALUE, pub value: VALUE, - pub _unused1: VALUE, - pub _unused2: VALUE, pub ic_cref: *const rb_cref_t, } #[repr(C)] @@ -330,6 +573,23 @@ pub struct iseq_inline_iv_cache_entry { pub struct iseq_inline_cvar_cache_entry { pub entry: *mut rb_cvar_class_tbl_entry, } +#[repr(C)] +#[repr(align(8))] +#[derive(Copy, Clone)] +pub struct iseq_inline_storage_entry { + pub _bindgen_opaque_blob: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_location_struct { + pub pathobj: VALUE, + pub base_label: VALUE, + pub label: VALUE, + pub first_lineno: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub code_location: rb_code_location_t, +} +pub type rb_iseq_location_t = rb_iseq_location_struct; +pub type iseq_bits_t = usize; pub const ISEQ_TYPE_TOP: rb_iseq_type = 0; pub const ISEQ_TYPE_METHOD: rb_iseq_type = 1; pub const ISEQ_TYPE_BLOCK: rb_iseq_type = 2; @@ -344,10 +604,617 @@ pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1; pub const BUILTIN_ATTR_SINGLE_NOARG_LEAF: rb_builtin_attr = 2; pub const BUILTIN_ATTR_INLINE_BLOCK: rb_builtin_attr = 4; pub const BUILTIN_ATTR_C_TRACE: rb_builtin_attr = 8; +pub const BUILTIN_ATTR_WITHOUT_INTERRUPTS: rb_builtin_attr = 16; pub type rb_builtin_attr = u32; +pub type rb_jit_func_t = ::std::option::Option< + unsafe extern "C" fn( + arg1: *mut rb_execution_context_struct, + arg2: *mut rb_control_frame_struct, + ) -> VALUE, +>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters { + pub flags: rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1, + pub size: ::std::os::raw::c_uint, + pub lead_num: ::std::os::raw::c_int, + pub opt_num: ::std::os::raw::c_int, + pub rest_start: ::std::os::raw::c_int, + pub post_start: ::std::os::raw::c_int, + pub post_num: ::std::os::raw::c_int, + pub block_start: ::std::os::raw::c_int, + pub opt_table: *const VALUE, + pub keyword: *const rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword, +} #[repr(C)] +#[repr(align(4))] #[derive(Debug, Copy, Clone)] -pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword { +pub struct rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 2usize]>, + pub __bindgen_padding_0: u16, +} +impl rb_iseq_constant_body_rb_iseq_parameters__bindgen_ty_1 { + #[inline] + pub fn has_lead(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_lead(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_lead_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_lead_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_opt(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_opt(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_opt_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_opt_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_post(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(3usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_post(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(3usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_post_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 3usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_post_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 3usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kw(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(4usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kw(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(4usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kw_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 4usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kw_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 4usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(5usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(5usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 5usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 5usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn has_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(6usize, 1u8) as u32) } + } + #[inline] + pub fn set_has_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(6usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn has_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 6usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_has_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 6usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ambiguous_param0(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(7usize, 1u8) as u32) } + } + #[inline] + pub fn set_ambiguous_param0(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(7usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ambiguous_param0_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 7usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ambiguous_param0_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 7usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn accepts_no_kwarg(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(8usize, 1u8) as u32) } + } + #[inline] + pub fn set_accepts_no_kwarg(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(8usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn accepts_no_kwarg_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 8usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_accepts_no_kwarg_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 8usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn ruby2_keywords(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(9usize, 1u8) as u32) } + } + #[inline] + pub fn set_ruby2_keywords(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(9usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn ruby2_keywords_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 9usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_ruby2_keywords_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 9usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_rest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(10usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_rest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(10usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_rest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 10usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_rest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 10usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn anon_kwrest(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(11usize, 1u8) as u32) } + } + #[inline] + pub fn set_anon_kwrest(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(11usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn anon_kwrest_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 11usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_anon_kwrest_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 11usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn use_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(12usize, 1u8) as u32) } + } + #[inline] + pub fn set_use_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(12usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn use_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 12usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_use_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 12usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn forwardable(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(13usize, 1u8) as u32) } + } + #[inline] + pub fn set_forwardable(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(13usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn forwardable_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 13usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_forwardable_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 13usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn accepts_no_block(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(14usize, 1u8) as u32) } + } + #[inline] + pub fn set_accepts_no_block(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(14usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn accepts_no_block_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 2usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 14usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_accepts_no_block_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 2usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 14usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + has_lead: ::std::os::raw::c_uint, + has_opt: ::std::os::raw::c_uint, + has_rest: ::std::os::raw::c_uint, + has_post: ::std::os::raw::c_uint, + has_kw: ::std::os::raw::c_uint, + has_kwrest: ::std::os::raw::c_uint, + has_block: ::std::os::raw::c_uint, + ambiguous_param0: ::std::os::raw::c_uint, + accepts_no_kwarg: ::std::os::raw::c_uint, + ruby2_keywords: ::std::os::raw::c_uint, + anon_rest: ::std::os::raw::c_uint, + anon_kwrest: ::std::os::raw::c_uint, + use_block: ::std::os::raw::c_uint, + forwardable: ::std::os::raw::c_uint, + accepts_no_block: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 2usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 2usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let has_lead: u32 = unsafe { ::std::mem::transmute(has_lead) }; + has_lead as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let has_opt: u32 = unsafe { ::std::mem::transmute(has_opt) }; + has_opt as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let has_rest: u32 = unsafe { ::std::mem::transmute(has_rest) }; + has_rest as u64 + }); + __bindgen_bitfield_unit.set(3usize, 1u8, { + let has_post: u32 = unsafe { ::std::mem::transmute(has_post) }; + has_post as u64 + }); + __bindgen_bitfield_unit.set(4usize, 1u8, { + let has_kw: u32 = unsafe { ::std::mem::transmute(has_kw) }; + has_kw as u64 + }); + __bindgen_bitfield_unit.set(5usize, 1u8, { + let has_kwrest: u32 = unsafe { ::std::mem::transmute(has_kwrest) }; + has_kwrest as u64 + }); + __bindgen_bitfield_unit.set(6usize, 1u8, { + let has_block: u32 = unsafe { ::std::mem::transmute(has_block) }; + has_block as u64 + }); + __bindgen_bitfield_unit.set(7usize, 1u8, { + let ambiguous_param0: u32 = unsafe { ::std::mem::transmute(ambiguous_param0) }; + ambiguous_param0 as u64 + }); + __bindgen_bitfield_unit.set(8usize, 1u8, { + let accepts_no_kwarg: u32 = unsafe { ::std::mem::transmute(accepts_no_kwarg) }; + accepts_no_kwarg as u64 + }); + __bindgen_bitfield_unit.set(9usize, 1u8, { + let ruby2_keywords: u32 = unsafe { ::std::mem::transmute(ruby2_keywords) }; + ruby2_keywords as u64 + }); + __bindgen_bitfield_unit.set(10usize, 1u8, { + let anon_rest: u32 = unsafe { ::std::mem::transmute(anon_rest) }; + anon_rest as u64 + }); + __bindgen_bitfield_unit.set(11usize, 1u8, { + let anon_kwrest: u32 = unsafe { ::std::mem::transmute(anon_kwrest) }; + anon_kwrest as u64 + }); + __bindgen_bitfield_unit.set(12usize, 1u8, { + let use_block: u32 = unsafe { ::std::mem::transmute(use_block) }; + use_block as u64 + }); + __bindgen_bitfield_unit.set(13usize, 1u8, { + let forwardable: u32 = unsafe { ::std::mem::transmute(forwardable) }; + forwardable as u64 + }); + __bindgen_bitfield_unit.set(14usize, 1u8, { + let accepts_no_block: u32 = unsafe { ::std::mem::transmute(accepts_no_block) }; + accepts_no_block as u64 + }); + __bindgen_bitfield_unit + } +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword { pub num: ::std::os::raw::c_int, pub required_num: ::std::os::raw::c_int, pub bits_start: ::std::os::raw::c_int, @@ -355,7 +1222,227 @@ pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword { pub table: *const ID, pub default_values: *mut VALUE, } +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_constant_body_iseq_insn_info { + pub body: *const iseq_insn_info_entry, + pub positions: *mut ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub succ_index_table: *mut succ_index_table, +} +pub const lvar_uninitialized: rb_iseq_constant_body_lvar_state = 0; +pub const lvar_initialized: rb_iseq_constant_body_lvar_state = 1; +pub const lvar_reassigned: rb_iseq_constant_body_lvar_state = 2; +pub type rb_iseq_constant_body_lvar_state = u32; +#[repr(C)] +pub struct rb_iseq_constant_body__bindgen_ty_1 { + pub flip_count: rb_snum_t, + pub script_lines: VALUE, + pub coverage: VALUE, + pub pc2branchindex: VALUE, + pub original_iseq: *mut VALUE, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union rb_iseq_constant_body__bindgen_ty_2 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +pub struct rb_iseq_struct { + pub flags: VALUE, + pub wrapper: VALUE, + pub body: *mut rb_iseq_constant_body, + pub aux: rb_iseq_struct__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1 { + pub compile_data: __BindgenUnionField<*mut iseq_compile_data>, + pub loader: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_1>, + pub exec: __BindgenUnionField<rb_iseq_struct__bindgen_ty_1__bindgen_ty_2>, + pub bindgen_union_field: [u64; 2usize], +} +#[repr(C)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_1 { + pub obj: VALUE, + pub index: ::std::os::raw::c_int, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct rb_iseq_struct__bindgen_ty_1__bindgen_ty_2 { + pub local_hooks_cnt: ::std::os::raw::c_uint, + pub global_trace_events: rb_event_flag_t, +} +#[repr(C)] +pub struct rb_captured_block { + pub self_: VALUE, + pub ep: *const VALUE, + pub code: rb_captured_block__bindgen_ty_1, +} +#[repr(C)] +pub struct rb_captured_block__bindgen_ty_1 { + pub iseq: __BindgenUnionField<*const rb_iseq_t>, + pub ifunc: __BindgenUnionField<*const vm_ifunc>, + pub val: __BindgenUnionField<VALUE>, + pub bindgen_union_field: u64, +} +pub const block_type_iseq: rb_block_type = 0; +pub const block_type_ifunc: rb_block_type = 1; +pub const block_type_symbol: rb_block_type = 2; +pub const block_type_proc: rb_block_type = 3; +pub type rb_block_type = u32; +#[repr(C)] +pub struct rb_block { + pub as_: rb_block__bindgen_ty_1, + pub type_: rb_block_type, +} +#[repr(C)] +pub struct rb_block__bindgen_ty_1 { + pub captured: __BindgenUnionField<rb_captured_block>, + pub symbol: __BindgenUnionField<VALUE>, + pub proc_: __BindgenUnionField<VALUE>, + pub bindgen_union_field: [u64; 3usize], +} +#[repr(C)] +pub struct rb_control_frame_struct { + pub pc: *const VALUE, + pub sp: *mut VALUE, + pub _iseq: *const rb_iseq_t, + pub self_: VALUE, + pub ep: *const VALUE, + pub block_code: *const ::std::os::raw::c_void, + pub jit_return: *mut ::std::os::raw::c_void, +} pub type rb_control_frame_t = rb_control_frame_struct; +#[repr(C)] +pub struct rb_proc_t { + pub block: rb_block, + pub _bitfield_align_1: [u8; 0], + pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>, + pub __bindgen_padding_0: [u8; 7usize], +} +impl rb_proc_t { + #[inline] + pub fn is_from_method(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(0usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_from_method(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(0usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_from_method_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 0usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_from_method_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 0usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_lambda(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(1usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_lambda(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(1usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_lambda_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 1usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_lambda_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 1usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn is_isolated(&self) -> ::std::os::raw::c_uint { + unsafe { ::std::mem::transmute(self._bitfield_1.get(2usize, 1u8) as u32) } + } + #[inline] + pub fn set_is_isolated(&mut self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + self._bitfield_1.set(2usize, 1u8, val as u64) + } + } + #[inline] + pub unsafe fn is_isolated_raw(this: *const Self) -> ::std::os::raw::c_uint { + unsafe { + ::std::mem::transmute(<__BindgenBitfieldUnit<[u8; 1usize]>>::raw_get( + ::std::ptr::addr_of!((*this)._bitfield_1), + 2usize, + 1u8, + ) as u32) + } + } + #[inline] + pub unsafe fn set_is_isolated_raw(this: *mut Self, val: ::std::os::raw::c_uint) { + unsafe { + let val: u32 = ::std::mem::transmute(val); + <__BindgenBitfieldUnit<[u8; 1usize]>>::raw_set( + ::std::ptr::addr_of_mut!((*this)._bitfield_1), + 2usize, + 1u8, + val as u64, + ) + } + } + #[inline] + pub fn new_bitfield_1( + is_from_method: ::std::os::raw::c_uint, + is_lambda: ::std::os::raw::c_uint, + is_isolated: ::std::os::raw::c_uint, + ) -> __BindgenBitfieldUnit<[u8; 1usize]> { + let mut __bindgen_bitfield_unit: __BindgenBitfieldUnit<[u8; 1usize]> = Default::default(); + __bindgen_bitfield_unit.set(0usize, 1u8, { + let is_from_method: u32 = unsafe { ::std::mem::transmute(is_from_method) }; + is_from_method as u64 + }); + __bindgen_bitfield_unit.set(1usize, 1u8, { + let is_lambda: u32 = unsafe { ::std::mem::transmute(is_lambda) }; + is_lambda as u64 + }); + __bindgen_bitfield_unit.set(2usize, 1u8, { + let is_isolated: u32 = unsafe { ::std::mem::transmute(is_isolated) }; + is_isolated as u64 + }); + __bindgen_bitfield_unit + } +} pub const VM_CHECKMATCH_TYPE_WHEN: vm_check_match_type = 1; pub const VM_CHECKMATCH_TYPE_CASE: vm_check_match_type = 2; pub const VM_CHECKMATCH_TYPE_RESCUE: vm_check_match_type = 3; @@ -391,17 +1478,43 @@ pub const VM_FRAME_FLAG_LAMBDA: vm_frame_env_flags = 256; pub const VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM: vm_frame_env_flags = 512; pub const VM_FRAME_FLAG_CFRAME_KW: vm_frame_env_flags = 1024; pub const VM_FRAME_FLAG_PASSED: vm_frame_env_flags = 2048; -pub const VM_FRAME_FLAG_NS_SWITCH: vm_frame_env_flags = 4096; -pub const VM_FRAME_FLAG_LOAD_ISEQ: vm_frame_env_flags = 8192; +pub const VM_FRAME_FLAG_BOX_REQUIRE: vm_frame_env_flags = 4096; pub const VM_ENV_FLAG_LOCAL: vm_frame_env_flags = 2; pub const VM_ENV_FLAG_ESCAPED: vm_frame_env_flags = 4; pub const VM_ENV_FLAG_WB_REQUIRED: vm_frame_env_flags = 8; pub const VM_ENV_FLAG_ISOLATED: vm_frame_env_flags = 16; pub type vm_frame_env_flags = u32; -pub type attr_index_t = u16; +pub type attr_index_t = u8; pub type shape_id_t = u32; +pub const SHAPE_ID_HEAP_INDEX_MASK: shape_id_fl_type = 7864320; +pub const SHAPE_ID_FL_COMPLEX: shape_id_fl_type = 8388608; +pub const SHAPE_ID_FL_FROZEN: shape_id_fl_type = 16777216; +pub const SHAPE_ID_FL_HAS_OBJECT_ID: shape_id_fl_type = 33554432; +pub const SHAPE_ID_LAYOUT_ROBJECT: shape_id_fl_type = 0; +pub const SHAPE_ID_LAYOUT_RCLASS: shape_id_fl_type = 67108864; +pub const SHAPE_ID_LAYOUT_RDATA: shape_id_fl_type = 134217728; +pub const SHAPE_ID_LAYOUT_OTHER: shape_id_fl_type = 201326592; +pub const SHAPE_ID_LAYOUT_MASK: shape_id_fl_type = 201326592; +pub const SHAPE_ID_FL_NON_CANONICAL_MASK: shape_id_fl_type = 50331648; +pub const SHAPE_ID_FLAGS_MASK: shape_id_fl_type = 267911168; +pub type shape_id_fl_type = u32; +pub const CONST_DEPRECATED: rb_const_flag_t = 256; +pub const CONST_VISIBILITY_MASK: rb_const_flag_t = 255; +pub const CONST_PUBLIC: rb_const_flag_t = 0; +pub const CONST_PRIVATE: rb_const_flag_t = 1; +pub const CONST_VISIBILITY_MAX: rb_const_flag_t = 2; +pub type rb_const_flag_t = u32; +#[repr(C)] +pub struct rb_const_entry_struct { + pub flag: rb_const_flag_t, + pub line: ::std::os::raw::c_int, + pub value: VALUE, + pub file: VALUE, +} +pub type rb_const_entry_t = rb_const_entry_struct; #[repr(C)] pub struct rb_cvar_class_tbl_entry { + pub imemo_flags: VALUE, pub index: u32, pub global_cvar_state: rb_serial_t, pub cref: *const rb_cref_t, @@ -426,7 +1539,7 @@ pub type vm_call_flag_bits = u32; #[repr(C)] pub struct rb_callinfo_kwarg { pub keyword_len: ::std::os::raw::c_int, - pub references: ::std::os::raw::c_int, + pub references: rb_atomic_t, pub keywords: __IncompleteArrayField<VALUE>, } #[repr(C)] @@ -434,8 +1547,8 @@ pub struct rb_callinfo { pub flags: VALUE, pub kwarg: *const rb_callinfo_kwarg, pub mid: VALUE, - pub flag: VALUE, - pub argc: VALUE, + pub flag: ::std::os::raw::c_uint, + pub argc: ::std::os::raw::c_uint, } #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -484,8 +1597,8 @@ pub const YARVINSN_putnil: ruby_vminsn_type = 17; pub const YARVINSN_putself: ruby_vminsn_type = 18; pub const YARVINSN_putobject: ruby_vminsn_type = 19; pub const YARVINSN_putspecialobject: ruby_vminsn_type = 20; -pub const YARVINSN_putstring: ruby_vminsn_type = 21; -pub const YARVINSN_putchilledstring: ruby_vminsn_type = 22; +pub const YARVINSN_dupstring: ruby_vminsn_type = 21; +pub const YARVINSN_dupchilledstring: ruby_vminsn_type = 22; pub const YARVINSN_concatstrings: ruby_vminsn_type = 23; pub const YARVINSN_anytostring: ruby_vminsn_type = 24; pub const YARVINSN_toregexp: ruby_vminsn_type = 25; @@ -539,172 +1652,255 @@ pub const YARVINSN_jump: ruby_vminsn_type = 72; pub const YARVINSN_branchif: ruby_vminsn_type = 73; pub const YARVINSN_branchunless: ruby_vminsn_type = 74; pub const YARVINSN_branchnil: ruby_vminsn_type = 75; -pub const YARVINSN_once: ruby_vminsn_type = 76; -pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 77; -pub const YARVINSN_opt_plus: ruby_vminsn_type = 78; -pub const YARVINSN_opt_minus: ruby_vminsn_type = 79; -pub const YARVINSN_opt_mult: ruby_vminsn_type = 80; -pub const YARVINSN_opt_div: ruby_vminsn_type = 81; -pub const YARVINSN_opt_mod: ruby_vminsn_type = 82; -pub const YARVINSN_opt_eq: ruby_vminsn_type = 83; -pub const YARVINSN_opt_neq: ruby_vminsn_type = 84; -pub const YARVINSN_opt_lt: ruby_vminsn_type = 85; -pub const YARVINSN_opt_le: ruby_vminsn_type = 86; -pub const YARVINSN_opt_gt: ruby_vminsn_type = 87; -pub const YARVINSN_opt_ge: ruby_vminsn_type = 88; -pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 89; -pub const YARVINSN_opt_and: ruby_vminsn_type = 90; -pub const YARVINSN_opt_or: ruby_vminsn_type = 91; -pub const YARVINSN_opt_aref: ruby_vminsn_type = 92; -pub const YARVINSN_opt_aset: ruby_vminsn_type = 93; -pub const YARVINSN_opt_aset_with: ruby_vminsn_type = 94; -pub const YARVINSN_opt_aref_with: ruby_vminsn_type = 95; -pub const YARVINSN_opt_length: ruby_vminsn_type = 96; -pub const YARVINSN_opt_size: ruby_vminsn_type = 97; -pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 98; -pub const YARVINSN_opt_succ: ruby_vminsn_type = 99; -pub const YARVINSN_opt_not: ruby_vminsn_type = 100; -pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 101; -pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 102; -pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 103; -pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 104; -pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 105; -pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 106; -pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 107; -pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 108; -pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 109; -pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 110; -pub const YARVINSN_trace_nop: ruby_vminsn_type = 111; -pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 112; -pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 113; -pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 114; -pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 115; -pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 116; -pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 117; -pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 118; -pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 119; -pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 120; -pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 121; -pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 122; -pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 123; -pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 124; -pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 125; -pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 126; -pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 127; -pub const YARVINSN_trace_putnil: ruby_vminsn_type = 128; -pub const YARVINSN_trace_putself: ruby_vminsn_type = 129; -pub const YARVINSN_trace_putobject: ruby_vminsn_type = 130; -pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 131; -pub const YARVINSN_trace_putstring: ruby_vminsn_type = 132; -pub const YARVINSN_trace_putchilledstring: ruby_vminsn_type = 133; -pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 134; -pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 135; -pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 136; -pub const YARVINSN_trace_intern: ruby_vminsn_type = 137; -pub const YARVINSN_trace_newarray: ruby_vminsn_type = 138; -pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 139; -pub const YARVINSN_trace_duparray: ruby_vminsn_type = 140; -pub const YARVINSN_trace_duphash: ruby_vminsn_type = 141; -pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 142; -pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 143; -pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 144; -pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 145; -pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 146; -pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 147; -pub const YARVINSN_trace_newhash: ruby_vminsn_type = 148; -pub const YARVINSN_trace_newrange: ruby_vminsn_type = 149; -pub const YARVINSN_trace_pop: ruby_vminsn_type = 150; -pub const YARVINSN_trace_dup: ruby_vminsn_type = 151; -pub const YARVINSN_trace_dupn: ruby_vminsn_type = 152; -pub const YARVINSN_trace_swap: ruby_vminsn_type = 153; -pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 154; -pub const YARVINSN_trace_topn: ruby_vminsn_type = 155; -pub const YARVINSN_trace_setn: ruby_vminsn_type = 156; -pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 157; -pub const YARVINSN_trace_defined: ruby_vminsn_type = 158; -pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 159; -pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 160; -pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 161; -pub const YARVINSN_trace_checktype: ruby_vminsn_type = 162; -pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 163; -pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 164; -pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 165; -pub const YARVINSN_trace_send: ruby_vminsn_type = 166; -pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 167; -pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 168; -pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 169; -pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 170; -pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 171; -pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 172; -pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 173; -pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 174; -pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 175; -pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 176; -pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 177; -pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 178; -pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 179; -pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 180; -pub const YARVINSN_trace_leave: ruby_vminsn_type = 181; -pub const YARVINSN_trace_throw: ruby_vminsn_type = 182; -pub const YARVINSN_trace_jump: ruby_vminsn_type = 183; -pub const YARVINSN_trace_branchif: ruby_vminsn_type = 184; -pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 185; -pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 186; -pub const YARVINSN_trace_once: ruby_vminsn_type = 187; -pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 188; -pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 189; -pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 190; -pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 191; -pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 192; -pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 193; -pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 194; -pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 195; -pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 196; -pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 197; -pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 198; -pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 199; -pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 200; -pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 201; -pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 202; -pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 203; -pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 204; -pub const YARVINSN_trace_opt_aset_with: ruby_vminsn_type = 205; -pub const YARVINSN_trace_opt_aref_with: ruby_vminsn_type = 206; -pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 207; -pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 208; -pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 209; -pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 210; -pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 211; -pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 212; -pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 213; -pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 214; -pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 215; -pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 216; -pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 217; -pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 218; -pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 219; -pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 220; -pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 221; -pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 222; -pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 223; -pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 224; -pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 225; -pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 226; -pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 227; -pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 228; -pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 229; -pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 230; -pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 231; -pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 232; -pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 233; -pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 234; -pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 235; -pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 236; -pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 237; +pub const YARVINSN_jump_without_ints: ruby_vminsn_type = 76; +pub const YARVINSN_branchif_without_ints: ruby_vminsn_type = 77; +pub const YARVINSN_branchunless_without_ints: ruby_vminsn_type = 78; +pub const YARVINSN_branchnil_without_ints: ruby_vminsn_type = 79; +pub const YARVINSN_once: ruby_vminsn_type = 80; +pub const YARVINSN_opt_case_dispatch: ruby_vminsn_type = 81; +pub const YARVINSN_opt_plus: ruby_vminsn_type = 82; +pub const YARVINSN_opt_minus: ruby_vminsn_type = 83; +pub const YARVINSN_opt_mult: ruby_vminsn_type = 84; +pub const YARVINSN_opt_div: ruby_vminsn_type = 85; +pub const YARVINSN_opt_mod: ruby_vminsn_type = 86; +pub const YARVINSN_opt_eq: ruby_vminsn_type = 87; +pub const YARVINSN_opt_neq: ruby_vminsn_type = 88; +pub const YARVINSN_opt_lt: ruby_vminsn_type = 89; +pub const YARVINSN_opt_le: ruby_vminsn_type = 90; +pub const YARVINSN_opt_gt: ruby_vminsn_type = 91; +pub const YARVINSN_opt_ge: ruby_vminsn_type = 92; +pub const YARVINSN_opt_ltlt: ruby_vminsn_type = 93; +pub const YARVINSN_opt_and: ruby_vminsn_type = 94; +pub const YARVINSN_opt_or: ruby_vminsn_type = 95; +pub const YARVINSN_opt_aref: ruby_vminsn_type = 96; +pub const YARVINSN_opt_aset: ruby_vminsn_type = 97; +pub const YARVINSN_opt_length: ruby_vminsn_type = 98; +pub const YARVINSN_opt_size: ruby_vminsn_type = 99; +pub const YARVINSN_opt_empty_p: ruby_vminsn_type = 100; +pub const YARVINSN_opt_succ: ruby_vminsn_type = 101; +pub const YARVINSN_opt_not: ruby_vminsn_type = 102; +pub const YARVINSN_opt_regexpmatch2: ruby_vminsn_type = 103; +pub const YARVINSN_invokebuiltin: ruby_vminsn_type = 104; +pub const YARVINSN_opt_invokebuiltin_delegate: ruby_vminsn_type = 105; +pub const YARVINSN_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 106; +pub const YARVINSN_getlocal_WC_0: ruby_vminsn_type = 107; +pub const YARVINSN_getlocal_WC_1: ruby_vminsn_type = 108; +pub const YARVINSN_setlocal_WC_0: ruby_vminsn_type = 109; +pub const YARVINSN_setlocal_WC_1: ruby_vminsn_type = 110; +pub const YARVINSN_putobject_INT2FIX_0_: ruby_vminsn_type = 111; +pub const YARVINSN_putobject_INT2FIX_1_: ruby_vminsn_type = 112; +pub const YARVINSN_trace_nop: ruby_vminsn_type = 113; +pub const YARVINSN_trace_getlocal: ruby_vminsn_type = 114; +pub const YARVINSN_trace_setlocal: ruby_vminsn_type = 115; +pub const YARVINSN_trace_getblockparam: ruby_vminsn_type = 116; +pub const YARVINSN_trace_setblockparam: ruby_vminsn_type = 117; +pub const YARVINSN_trace_getblockparamproxy: ruby_vminsn_type = 118; +pub const YARVINSN_trace_getspecial: ruby_vminsn_type = 119; +pub const YARVINSN_trace_setspecial: ruby_vminsn_type = 120; +pub const YARVINSN_trace_getinstancevariable: ruby_vminsn_type = 121; +pub const YARVINSN_trace_setinstancevariable: ruby_vminsn_type = 122; +pub const YARVINSN_trace_getclassvariable: ruby_vminsn_type = 123; +pub const YARVINSN_trace_setclassvariable: ruby_vminsn_type = 124; +pub const YARVINSN_trace_opt_getconstant_path: ruby_vminsn_type = 125; +pub const YARVINSN_trace_getconstant: ruby_vminsn_type = 126; +pub const YARVINSN_trace_setconstant: ruby_vminsn_type = 127; +pub const YARVINSN_trace_getglobal: ruby_vminsn_type = 128; +pub const YARVINSN_trace_setglobal: ruby_vminsn_type = 129; +pub const YARVINSN_trace_putnil: ruby_vminsn_type = 130; +pub const YARVINSN_trace_putself: ruby_vminsn_type = 131; +pub const YARVINSN_trace_putobject: ruby_vminsn_type = 132; +pub const YARVINSN_trace_putspecialobject: ruby_vminsn_type = 133; +pub const YARVINSN_trace_dupstring: ruby_vminsn_type = 134; +pub const YARVINSN_trace_dupchilledstring: ruby_vminsn_type = 135; +pub const YARVINSN_trace_concatstrings: ruby_vminsn_type = 136; +pub const YARVINSN_trace_anytostring: ruby_vminsn_type = 137; +pub const YARVINSN_trace_toregexp: ruby_vminsn_type = 138; +pub const YARVINSN_trace_intern: ruby_vminsn_type = 139; +pub const YARVINSN_trace_newarray: ruby_vminsn_type = 140; +pub const YARVINSN_trace_pushtoarraykwsplat: ruby_vminsn_type = 141; +pub const YARVINSN_trace_duparray: ruby_vminsn_type = 142; +pub const YARVINSN_trace_duphash: ruby_vminsn_type = 143; +pub const YARVINSN_trace_expandarray: ruby_vminsn_type = 144; +pub const YARVINSN_trace_concatarray: ruby_vminsn_type = 145; +pub const YARVINSN_trace_concattoarray: ruby_vminsn_type = 146; +pub const YARVINSN_trace_pushtoarray: ruby_vminsn_type = 147; +pub const YARVINSN_trace_splatarray: ruby_vminsn_type = 148; +pub const YARVINSN_trace_splatkw: ruby_vminsn_type = 149; +pub const YARVINSN_trace_newhash: ruby_vminsn_type = 150; +pub const YARVINSN_trace_newrange: ruby_vminsn_type = 151; +pub const YARVINSN_trace_pop: ruby_vminsn_type = 152; +pub const YARVINSN_trace_dup: ruby_vminsn_type = 153; +pub const YARVINSN_trace_dupn: ruby_vminsn_type = 154; +pub const YARVINSN_trace_swap: ruby_vminsn_type = 155; +pub const YARVINSN_trace_opt_reverse: ruby_vminsn_type = 156; +pub const YARVINSN_trace_topn: ruby_vminsn_type = 157; +pub const YARVINSN_trace_setn: ruby_vminsn_type = 158; +pub const YARVINSN_trace_adjuststack: ruby_vminsn_type = 159; +pub const YARVINSN_trace_defined: ruby_vminsn_type = 160; +pub const YARVINSN_trace_definedivar: ruby_vminsn_type = 161; +pub const YARVINSN_trace_checkmatch: ruby_vminsn_type = 162; +pub const YARVINSN_trace_checkkeyword: ruby_vminsn_type = 163; +pub const YARVINSN_trace_checktype: ruby_vminsn_type = 164; +pub const YARVINSN_trace_defineclass: ruby_vminsn_type = 165; +pub const YARVINSN_trace_definemethod: ruby_vminsn_type = 166; +pub const YARVINSN_trace_definesmethod: ruby_vminsn_type = 167; +pub const YARVINSN_trace_send: ruby_vminsn_type = 168; +pub const YARVINSN_trace_sendforward: ruby_vminsn_type = 169; +pub const YARVINSN_trace_opt_send_without_block: ruby_vminsn_type = 170; +pub const YARVINSN_trace_opt_new: ruby_vminsn_type = 171; +pub const YARVINSN_trace_objtostring: ruby_vminsn_type = 172; +pub const YARVINSN_trace_opt_ary_freeze: ruby_vminsn_type = 173; +pub const YARVINSN_trace_opt_hash_freeze: ruby_vminsn_type = 174; +pub const YARVINSN_trace_opt_str_freeze: ruby_vminsn_type = 175; +pub const YARVINSN_trace_opt_nil_p: ruby_vminsn_type = 176; +pub const YARVINSN_trace_opt_str_uminus: ruby_vminsn_type = 177; +pub const YARVINSN_trace_opt_duparray_send: ruby_vminsn_type = 178; +pub const YARVINSN_trace_opt_newarray_send: ruby_vminsn_type = 179; +pub const YARVINSN_trace_invokesuper: ruby_vminsn_type = 180; +pub const YARVINSN_trace_invokesuperforward: ruby_vminsn_type = 181; +pub const YARVINSN_trace_invokeblock: ruby_vminsn_type = 182; +pub const YARVINSN_trace_leave: ruby_vminsn_type = 183; +pub const YARVINSN_trace_throw: ruby_vminsn_type = 184; +pub const YARVINSN_trace_jump: ruby_vminsn_type = 185; +pub const YARVINSN_trace_branchif: ruby_vminsn_type = 186; +pub const YARVINSN_trace_branchunless: ruby_vminsn_type = 187; +pub const YARVINSN_trace_branchnil: ruby_vminsn_type = 188; +pub const YARVINSN_trace_jump_without_ints: ruby_vminsn_type = 189; +pub const YARVINSN_trace_branchif_without_ints: ruby_vminsn_type = 190; +pub const YARVINSN_trace_branchunless_without_ints: ruby_vminsn_type = 191; +pub const YARVINSN_trace_branchnil_without_ints: ruby_vminsn_type = 192; +pub const YARVINSN_trace_once: ruby_vminsn_type = 193; +pub const YARVINSN_trace_opt_case_dispatch: ruby_vminsn_type = 194; +pub const YARVINSN_trace_opt_plus: ruby_vminsn_type = 195; +pub const YARVINSN_trace_opt_minus: ruby_vminsn_type = 196; +pub const YARVINSN_trace_opt_mult: ruby_vminsn_type = 197; +pub const YARVINSN_trace_opt_div: ruby_vminsn_type = 198; +pub const YARVINSN_trace_opt_mod: ruby_vminsn_type = 199; +pub const YARVINSN_trace_opt_eq: ruby_vminsn_type = 200; +pub const YARVINSN_trace_opt_neq: ruby_vminsn_type = 201; +pub const YARVINSN_trace_opt_lt: ruby_vminsn_type = 202; +pub const YARVINSN_trace_opt_le: ruby_vminsn_type = 203; +pub const YARVINSN_trace_opt_gt: ruby_vminsn_type = 204; +pub const YARVINSN_trace_opt_ge: ruby_vminsn_type = 205; +pub const YARVINSN_trace_opt_ltlt: ruby_vminsn_type = 206; +pub const YARVINSN_trace_opt_and: ruby_vminsn_type = 207; +pub const YARVINSN_trace_opt_or: ruby_vminsn_type = 208; +pub const YARVINSN_trace_opt_aref: ruby_vminsn_type = 209; +pub const YARVINSN_trace_opt_aset: ruby_vminsn_type = 210; +pub const YARVINSN_trace_opt_length: ruby_vminsn_type = 211; +pub const YARVINSN_trace_opt_size: ruby_vminsn_type = 212; +pub const YARVINSN_trace_opt_empty_p: ruby_vminsn_type = 213; +pub const YARVINSN_trace_opt_succ: ruby_vminsn_type = 214; +pub const YARVINSN_trace_opt_not: ruby_vminsn_type = 215; +pub const YARVINSN_trace_opt_regexpmatch2: ruby_vminsn_type = 216; +pub const YARVINSN_trace_invokebuiltin: ruby_vminsn_type = 217; +pub const YARVINSN_trace_opt_invokebuiltin_delegate: ruby_vminsn_type = 218; +pub const YARVINSN_trace_opt_invokebuiltin_delegate_leave: ruby_vminsn_type = 219; +pub const YARVINSN_trace_getlocal_WC_0: ruby_vminsn_type = 220; +pub const YARVINSN_trace_getlocal_WC_1: ruby_vminsn_type = 221; +pub const YARVINSN_trace_setlocal_WC_0: ruby_vminsn_type = 222; +pub const YARVINSN_trace_setlocal_WC_1: ruby_vminsn_type = 223; +pub const YARVINSN_trace_putobject_INT2FIX_0_: ruby_vminsn_type = 224; +pub const YARVINSN_trace_putobject_INT2FIX_1_: ruby_vminsn_type = 225; +pub const YARVINSN_zjit_getblockparamproxy: ruby_vminsn_type = 226; +pub const YARVINSN_zjit_getinstancevariable: ruby_vminsn_type = 227; +pub const YARVINSN_zjit_setinstancevariable: ruby_vminsn_type = 228; +pub const YARVINSN_zjit_splatkw: ruby_vminsn_type = 229; +pub const YARVINSN_zjit_definedivar: ruby_vminsn_type = 230; +pub const YARVINSN_zjit_send: ruby_vminsn_type = 231; +pub const YARVINSN_zjit_opt_send_without_block: ruby_vminsn_type = 232; +pub const YARVINSN_zjit_objtostring: ruby_vminsn_type = 233; +pub const YARVINSN_zjit_opt_nil_p: ruby_vminsn_type = 234; +pub const YARVINSN_zjit_invokesuper: ruby_vminsn_type = 235; +pub const YARVINSN_zjit_invokeblock: ruby_vminsn_type = 236; +pub const YARVINSN_zjit_opt_plus: ruby_vminsn_type = 237; +pub const YARVINSN_zjit_opt_minus: ruby_vminsn_type = 238; +pub const YARVINSN_zjit_opt_mult: ruby_vminsn_type = 239; +pub const YARVINSN_zjit_opt_div: ruby_vminsn_type = 240; +pub const YARVINSN_zjit_opt_mod: ruby_vminsn_type = 241; +pub const YARVINSN_zjit_opt_eq: ruby_vminsn_type = 242; +pub const YARVINSN_zjit_opt_neq: ruby_vminsn_type = 243; +pub const YARVINSN_zjit_opt_lt: ruby_vminsn_type = 244; +pub const YARVINSN_zjit_opt_le: ruby_vminsn_type = 245; +pub const YARVINSN_zjit_opt_gt: ruby_vminsn_type = 246; +pub const YARVINSN_zjit_opt_ge: ruby_vminsn_type = 247; +pub const YARVINSN_zjit_opt_ltlt: ruby_vminsn_type = 248; +pub const YARVINSN_zjit_opt_and: ruby_vminsn_type = 249; +pub const YARVINSN_zjit_opt_or: ruby_vminsn_type = 250; +pub const YARVINSN_zjit_opt_aref: ruby_vminsn_type = 251; +pub const YARVINSN_zjit_opt_aset: ruby_vminsn_type = 252; +pub const YARVINSN_zjit_opt_length: ruby_vminsn_type = 253; +pub const YARVINSN_zjit_opt_size: ruby_vminsn_type = 254; +pub const YARVINSN_zjit_opt_empty_p: ruby_vminsn_type = 255; +pub const YARVINSN_zjit_opt_succ: ruby_vminsn_type = 256; +pub const YARVINSN_zjit_opt_not: ruby_vminsn_type = 257; +pub const YARVINSN_zjit_opt_regexpmatch2: ruby_vminsn_type = 258; +pub const VM_INSTRUCTION_SIZE: ruby_vminsn_type = 259; pub type ruby_vminsn_type = u32; pub type rb_iseq_callback = ::std::option::Option< unsafe extern "C" fn(arg1: *const rb_iseq_t, arg2: *mut ::std::os::raw::c_void), >; +#[repr(C)] +#[repr(align(8))] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data { + pub _bindgen_opaque_blob: [u64; 24usize], +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union iseq_compile_data__bindgen_ty_1 { + pub list: *mut iseq_bits_t, + pub single: iseq_bits_t, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_2 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_compile_data__bindgen_ty_3 { + pub storage_head: *mut iseq_compile_data_storage, + pub storage_current: *mut iseq_compile_data_storage, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_insn_info_entry { + pub line_no: ::std::os::raw::c_int, + pub node_id: ::std::os::raw::c_int, + pub events: rb_event_flag_t, +} +pub const CATCH_TYPE_RESCUE: rb_catch_type = 3; +pub const CATCH_TYPE_ENSURE: rb_catch_type = 5; +pub const CATCH_TYPE_RETRY: rb_catch_type = 7; +pub const CATCH_TYPE_BREAK: rb_catch_type = 9; +pub const CATCH_TYPE_REDO: rb_catch_type = 11; +pub const CATCH_TYPE_NEXT: rb_catch_type = 13; +pub type rb_catch_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct iseq_catch_table_entry { + pub type_: rb_catch_type, + pub iseq: *mut rb_iseq_t, + pub start: ::std::os::raw::c_uint, + pub end: ::std::os::raw::c_uint, + pub cont: ::std::os::raw::c_uint, + pub sp: ::std::os::raw::c_uint, +} +#[repr(C, packed)] +pub struct iseq_catch_table { + pub size: ::std::os::raw::c_uint, + pub entries: __IncompleteArrayField<iseq_catch_table_entry>, +} +#[repr(C)] +#[derive(Debug)] +pub struct iseq_compile_data_storage { + pub next: *mut iseq_compile_data_storage, + pub pos: ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub buff: __IncompleteArrayField<::std::os::raw::c_char>, +} pub const DEFINED_NOT_DEFINED: defined_type = 0; pub const DEFINED_NIL: defined_type = 1; pub const DEFINED_IVAR: defined_type = 2; @@ -724,9 +1920,35 @@ pub const DEFINED_REF: defined_type = 15; pub const DEFINED_FUNC: defined_type = 16; pub const DEFINED_CONST_FROM: defined_type = 17; pub type defined_type = u32; -pub const RB_INVALID_SHAPE_ID: _bindgen_ty_38 = 4294967295; -pub type _bindgen_ty_38 = u32; -pub type rb_iseq_param_keyword_struct = rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct zjit_jit_frame { + pub pc: *const VALUE, + pub iseq: *const rb_iseq_t, + pub materialize_block_code: bool, +} +pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; +pub type zjit_struct_offsets = u32; +pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; +pub const ROBJECT_OFFSET_AS_ARY: jit_bindgen_constants = 16; +pub const RCLASS_OFFSET_PRIME_FIELDS_OBJ: jit_bindgen_constants = 40; +pub const TDATA_OFFSET_FIELDS_OBJ: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_RSTRING_LEN: jit_bindgen_constants = 16; +pub const RB_SHAPE_FLAG_SHIFT: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_CFP: jit_bindgen_constants = 16; +pub const RUBY_OFFSET_EC_INTERRUPT_FLAG: jit_bindgen_constants = 32; +pub const RUBY_OFFSET_EC_INTERRUPT_MASK: jit_bindgen_constants = 36; +pub const RUBY_OFFSET_EC_THREAD_PTR: jit_bindgen_constants = 48; +pub const RUBY_OFFSET_EC_RACTOR_ID: jit_bindgen_constants = 64; +pub type jit_bindgen_constants = u32; +pub const rb_invalid_shape_id: shape_id_t = 524287; +pub type rb_iseq_param_keyword_struct = + rb_iseq_constant_body_rb_iseq_parameters_rb_iseq_param_keyword; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct succ_index_table { + pub _address: u8, +} unsafe extern "C" { pub fn ruby_xfree(ptr: *mut ::std::os::raw::c_void); pub fn rb_class_attached_object(klass: VALUE) -> VALUE; @@ -738,9 +1960,11 @@ unsafe extern "C" { pub fn rb_gc_mark(obj: VALUE); pub fn rb_gc_mark_movable(obj: VALUE); pub fn rb_gc_location(obj: VALUE) -> VALUE; + pub fn rb_gc_enable() -> VALUE; + pub fn rb_gc_disable() -> VALUE; + pub fn rb_gc_register_mark_object(object: VALUE); pub fn rb_gc_writebarrier(old: VALUE, young: VALUE); pub fn rb_class_get_superclass(klass: VALUE) -> VALUE; - pub static mut rb_cObject: VALUE; pub fn rb_funcallv( recv: VALUE, mid: ID, @@ -749,6 +1973,7 @@ unsafe extern "C" { ) -> VALUE; pub static mut rb_mKernel: VALUE; pub static mut rb_cBasicObject: VALUE; + pub static mut rb_cObject: VALUE; pub static mut rb_cArray: VALUE; pub static mut rb_cClass: VALUE; pub static mut rb_cFalseClass: VALUE; @@ -784,7 +2009,10 @@ unsafe extern "C" { pub fn rb_ary_resurrect(ary: VALUE) -> VALUE; pub fn rb_ary_cat(ary: VALUE, train: *const VALUE, len: ::std::os::raw::c_long) -> VALUE; pub fn rb_ary_push(ary: VALUE, elem: VALUE) -> VALUE; + pub fn rb_ary_pop(ary: VALUE) -> VALUE; + pub fn rb_ary_entry(ary: VALUE, off: ::std::os::raw::c_long) -> VALUE; pub fn rb_ary_clear(ary: VALUE) -> VALUE; + pub fn rb_ary_concat(lhs: VALUE, rhs: VALUE) -> VALUE; pub fn rb_hash_new() -> VALUE; pub fn rb_hash_aref(hash: VALUE, key: VALUE) -> VALUE; pub fn rb_hash_aset(hash: VALUE, key: VALUE, val: VALUE) -> VALUE; @@ -802,8 +2030,15 @@ unsafe extern "C" { pub fn rb_id2str(id: ID) -> VALUE; pub fn rb_sym2str(symbol: VALUE) -> VALUE; pub fn rb_class2name(klass: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_class_new_instance_pass_kw( + argc: ::std::os::raw::c_int, + argv: *const VALUE, + klass: VALUE, + ) -> VALUE; pub fn rb_obj_is_kind_of(obj: VALUE, klass: VALUE) -> VALUE; + pub fn rb_obj_alloc(klass: VALUE) -> VALUE; pub fn rb_obj_frozen_p(obj: VALUE) -> VALUE; + pub fn rb_class_real(klass: VALUE) -> VALUE; pub fn rb_class_inherited_p(scion: VALUE, ascendant: VALUE) -> VALUE; pub fn rb_backref_get() -> VALUE; pub fn rb_range_new(beg: VALUE, end: VALUE, excl: ::std::os::raw::c_int) -> VALUE; @@ -824,10 +2059,14 @@ unsafe extern "C" { pub fn rb_ivar_set(obj: VALUE, name: ID, val: VALUE) -> VALUE; pub fn rb_ivar_defined(obj: VALUE, name: ID) -> VALUE; pub fn rb_attr_get(obj: VALUE, name: ID) -> VALUE; - pub fn rb_obj_info_dump(obj: VALUE); + pub fn rb_const_get(space: VALUE, name: ID) -> VALUE; pub fn rb_class_allocate_instance(klass: VALUE) -> VALUE; pub fn rb_obj_equal(obj1: VALUE, obj2: VALUE) -> VALUE; - pub fn rb_reg_new_ary(ary: VALUE, options: ::std::os::raw::c_int) -> VALUE; + pub fn rb_reg_new_from_values( + cnt: ::std::os::raw::c_long, + elements: *const VALUE, + opt: ::std::os::raw::c_int, + ) -> VALUE; pub fn rb_ary_tmp_new_from_values( arg1: VALUE, arg2: ::std::os::raw::c_long, @@ -839,7 +2078,7 @@ unsafe extern "C" { elts: *const VALUE, ) -> VALUE; pub fn rb_vm_top_self() -> VALUE; - pub static mut rb_vm_insns_count: u64; + pub static mut rb_vm_insn_count: u64; pub fn rb_method_entry_at(obj: VALUE, id: ID) -> *const rb_method_entry_t; pub fn rb_callable_method_entry(klass: VALUE, id: ID) -> *const rb_callable_method_entry_t; pub fn rb_callable_method_entry_or_negative( @@ -847,6 +2086,7 @@ unsafe extern "C" { id: ID, ) -> *const rb_callable_method_entry_t; pub static mut rb_cISeq: VALUE; + pub static mut rb_cRubyVM: VALUE; pub static mut rb_mRubyVMFrozenCore: VALUE; pub static mut rb_block_param_proxy: VALUE; pub fn rb_vm_ep_local_ep(ep: *const VALUE) -> *const VALUE; @@ -857,12 +2097,23 @@ unsafe extern "C" { cfp: *const rb_control_frame_t, ) -> *const rb_callable_method_entry_t; pub fn rb_obj_info(obj: VALUE) -> *const ::std::os::raw::c_char; + pub fn rb_raw_obj_info( + buff: *mut ::std::os::raw::c_char, + buff_size: usize, + obj: VALUE, + ) -> *const ::std::os::raw::c_char; pub fn rb_ec_stack_check(ec: *mut rb_execution_context_struct) -> ::std::os::raw::c_int; pub fn rb_gc_writebarrier_remember(obj: VALUE); pub fn rb_shape_id_offset() -> i32; pub fn rb_obj_shape_id(obj: VALUE) -> shape_id_t; pub fn rb_shape_get_iv_index(shape_id: shape_id_t, id: ID, value: *mut attr_index_t) -> bool; - pub fn rb_shape_transition_add_ivar_no_warnings(obj: VALUE, id: ID) -> shape_id_t; + pub fn rb_shape_transition_add_ivar_no_warnings( + shape_id: shape_id_t, + id: ID, + klass: VALUE, + ) -> shape_id_t; + pub fn rb_const_lookup(klass: VALUE, id: ID) -> *mut rb_const_entry_t; + pub fn rb_ivar_get_at_no_ractor_check(obj: VALUE, index: attr_index_t) -> VALUE; pub fn rb_gvar_get(arg1: ID) -> VALUE; pub fn rb_gvar_set(arg1: ID, arg2: VALUE) -> VALUE; pub fn rb_ensure_iv_list_size(obj: VALUE, current_len: u32, newsize: u32); @@ -888,6 +2139,7 @@ unsafe extern "C" { arg: st_data_t, ) -> ::std::os::raw::c_int; pub fn rb_hash_new_with_size(size: st_index_t) -> VALUE; + pub fn rb_hash_new_with_bulk_insert(argc: ::std::os::raw::c_long, argv: *const VALUE) -> VALUE; pub fn rb_hash_resurrect(hash: VALUE) -> VALUE; pub fn rb_hash_stlike_lookup( hash: VALUE, @@ -900,6 +2152,7 @@ unsafe extern "C" { pub fn rb_float_minus(x: VALUE, y: VALUE) -> VALUE; pub fn rb_float_mul(x: VALUE, y: VALUE) -> VALUE; pub fn rb_float_div(x: VALUE, y: VALUE) -> VALUE; + pub fn rb_flo_to_i(num: VALUE) -> VALUE; pub fn rb_fix_aref(fix: VALUE, idx: VALUE) -> VALUE; pub fn rb_vm_insn_addr2opcode(addr: *const ::std::os::raw::c_void) -> ::std::os::raw::c_int; pub fn rb_iseq_line_no(iseq: *const rb_iseq_t, pos: usize) -> ::std::os::raw::c_uint; @@ -912,30 +2165,13 @@ unsafe extern "C" { buff: *mut VALUE, lines: *mut ::std::os::raw::c_int, ) -> ::std::os::raw::c_int; + pub fn rb_profile_frame_path(frame: VALUE) -> VALUE; + pub fn rb_profile_frame_absolute_path(frame: VALUE) -> VALUE; + pub fn rb_profile_frame_full_label(frame: VALUE) -> VALUE; pub fn rb_jit_cont_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); - pub fn rb_zjit_get_page_size() -> u32; - pub fn rb_zjit_reserve_addr_space(mem_size: u32) -> *mut u8; pub fn rb_zjit_profile_disable(iseq: *const rb_iseq_t); pub fn rb_vm_base_ptr(cfp: *mut rb_control_frame_struct) -> *mut VALUE; - pub fn rb_zjit_multi_ractor_p() -> bool; pub fn rb_zjit_constcache_shareable(ice: *const iseq_inline_constant_cache_entry) -> bool; - pub fn rb_zjit_vm_unlock( - recursive_lock_level: *mut ::std::os::raw::c_uint, - file: *const ::std::os::raw::c_char, - line: ::std::os::raw::c_int, - ); - pub fn rb_zjit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); - pub fn rb_zjit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; - pub fn rb_zjit_icache_invalidate( - start: *mut ::std::os::raw::c_void, - end: *mut ::std::os::raw::c_void, - ); - pub fn rb_zjit_vm_lock_then_barrier( - recursive_lock_level: *mut ::std::os::raw::c_uint, - file: *const ::std::os::raw::c_char, - line: ::std::os::raw::c_int, - ); pub fn rb_zjit_iseq_insn_set( iseq: *const rb_iseq_t, insn_idx: ::std::os::raw::c_uint, @@ -944,11 +2180,33 @@ unsafe extern "C" { pub fn rb_iseq_get_zjit_payload(iseq: *const rb_iseq_t) -> *mut ::std::os::raw::c_void; pub fn rb_iseq_set_zjit_payload(iseq: *const rb_iseq_t, payload: *mut ::std::os::raw::c_void); pub fn rb_zjit_print_exception(); - pub fn rb_zjit_shape_obj_too_complex_p(obj: VALUE) -> bool; pub fn rb_zjit_singleton_class_p(klass: VALUE) -> bool; + pub fn rb_zjit_defined_ivar(obj: VALUE, id: ID, pushval: VALUE) -> VALUE; + pub fn rb_zjit_method_tracing_currently_enabled() -> bool; + pub fn rb_zjit_iseq_tracing_currently_enabled() -> bool; + pub fn rb_zjit_insn_leaf(insn: ::std::os::raw::c_int, opes: *const VALUE) -> bool; + pub fn rb_zjit_local_id(iseq: *const rb_iseq_t, idx: ::std::os::raw::c_uint) -> ID; + pub fn rb_zjit_cme_is_cfunc( + me: *const rb_callable_method_entry_t, + func: *const ::std::os::raw::c_void, + ) -> bool; + pub fn rb_zjit_vm_search_method( + cd_owner: VALUE, + cd: *mut rb_call_data, + recv: VALUE, + ) -> *const rb_callable_method_entry_struct; + pub fn rb_zjit_class_initialized_p(klass: VALUE) -> bool; + pub fn rb_zjit_class_get_alloc_func(klass: VALUE) -> rb_alloc_func_t; + pub fn rb_zjit_class_has_default_allocator(klass: VALUE) -> bool; + pub fn rb_vm_untag_block_handler(block_handler: VALUE) -> VALUE; + pub fn rb_vm_get_untagged_block_handler(reg_cfp: *mut rb_control_frame_t) -> VALUE; pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE; pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int; + pub fn rb_iseq_bare_opcode_at_pc( + iseq: *const rb_iseq_t, + pc: *const VALUE, + ) -> ::std::os::raw::c_int; pub fn rb_RSTRING_LEN(str_: VALUE) -> ::std::os::raw::c_ulong; pub fn rb_RSTRING_PTR(str_: VALUE) -> *mut ::std::os::raw::c_char; pub fn rb_insn_name(insn: VALUE) -> *const ::std::os::raw::c_char; @@ -975,6 +2233,17 @@ unsafe extern "C" { ) -> *mut rb_method_cfunc_t; pub fn rb_get_def_method_serial(def: *const rb_method_definition_t) -> usize; pub fn rb_get_def_original_id(def: *const rb_method_definition_t) -> ID; + pub fn rb_get_def_bmethod_proc(def: *mut rb_method_definition_t) -> VALUE; + pub fn rb_jit_get_proc_ptr(procv: VALUE) -> *mut rb_proc_t; + pub fn rb_optimized_call( + recv: VALUE, + ec: *mut rb_execution_context_t, + argc: ::std::os::raw::c_int, + argv: *mut VALUE, + kw_splat: ::std::os::raw::c_int, + block_handler: VALUE, + ) -> VALUE; + pub fn rb_jit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_get_mct_argc(mct: *const rb_method_cfunc_t) -> ::std::os::raw::c_int; pub fn rb_get_mct_func(mct: *const rb_method_cfunc_t) -> *mut ::std::os::raw::c_void; pub fn rb_get_def_iseq_ptr(def: *mut rb_method_definition_t) -> *const rb_iseq_t; @@ -1014,7 +2283,6 @@ unsafe extern "C" { pub fn rb_FL_TEST(obj: VALUE, flags: VALUE) -> VALUE; pub fn rb_FL_TEST_RAW(obj: VALUE, flags: VALUE) -> VALUE; pub fn rb_RB_TYPE_P(obj: VALUE, t: ruby_value_type) -> bool; - pub fn rb_RSTRUCT_LEN(st: VALUE) -> ::std::os::raw::c_long; pub fn rb_get_call_data_ci(cd: *const rb_call_data) -> *const rb_callinfo; pub fn rb_BASIC_OP_UNREDEFINED_P(bop: ruby_basic_operators, klass: u32) -> bool; pub fn rb_RCLASS_ORIGIN(c: VALUE) -> VALUE; @@ -1023,6 +2291,36 @@ unsafe extern "C" { pub fn rb_IMEMO_TYPE_P(imemo: VALUE, imemo_type: imemo_type) -> ::std::os::raw::c_int; pub fn rb_assert_cme_handle(handle: VALUE); pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; + pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); + pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool; + pub fn rb_jit_multi_ractor_p() -> bool; + pub fn rb_jit_class_fields_embedded_p(klass: VALUE) -> bool; + pub fn rb_jit_data_fields_embedded_p(obj: VALUE) -> bool; + pub fn rb_jit_vm_lock_then_barrier( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_jit_vm_unlock( + recursive_lock_level: *mut ::std::os::raw::c_uint, + file: *const ::std::os::raw::c_char, + line: ::std::os::raw::c_int, + ); + pub fn rb_iseq_reset_jit_func(iseq: *const rb_iseq_t); + pub fn rb_jit_get_page_size() -> u32; + pub fn rb_jit_reserve_addr_space(mem_size: u32) -> *mut u8; + pub fn rb_jit_for_each_iseq(callback: rb_iseq_callback, data: *mut ::std::os::raw::c_void); + pub fn rb_jit_mark_writable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32); + pub fn rb_jit_mark_unused(mem_block: *mut ::std::os::raw::c_void, mem_size: u32) -> bool; + pub fn rb_jit_icache_invalidate( + start: *mut ::std::os::raw::c_void, + end: *mut ::std::os::raw::c_void, + ); + pub fn rb_jit_fix_div_fix(recv: VALUE, obj: VALUE) -> VALUE; + pub fn rb_yarv_str_eql_internal(str1: VALUE, str2: VALUE) -> VALUE; + pub fn rb_jit_str_concat_codepoint(str_: VALUE, codepoint: VALUE); + pub fn rb_jit_shape_capacity(shape_id: shape_id_t) -> attr_index_t; } diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index c9ebcebc86..05b0055032 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -12,6 +12,15 @@ use crate::cruby::*; use std::collections::HashMap; use std::ffi::c_void; use crate::hir_type::{types, Type}; +use crate::hir::{self, FieldName}; + +// Array iteration builtin functions (defined in array.c) +unsafe extern "C" { + fn rb_jit_ary_at_end(ec: EcPtr, self_: VALUE, index: VALUE) -> VALUE; + fn rb_jit_ary_at(ec: EcPtr, self_: VALUE, index: VALUE) -> VALUE; + fn rb_jit_fixnum_inc(ec: EcPtr, self_: VALUE, num: VALUE) -> VALUE; + fn rb_str_equal(str1: VALUE, str2: VALUE) -> VALUE; +} pub struct Annotations { cfuncs: HashMap<*mut c_void, FnProperties>, @@ -29,6 +38,20 @@ pub struct FnProperties { pub return_type: Type, /// Whether it's legal to remove the call if the result is unused pub elidable: bool, + pub inline: fn(&mut hir::Function, hir::BlockId, hir::InsnId, &[hir::InsnId], hir::InsnId) -> Option<hir::InsnId>, +} + +/// A safe default for un-annotated Ruby methods: we can't optimize them or their returned values. +impl Default for FnProperties { + fn default() -> Self { + Self { + no_gc: false, + leaf: false, + return_type: types::BasicObject, + elidable: false, + inline: no_inline, + } + } } impl Annotations { @@ -113,7 +136,7 @@ fn annotate_builtin_method(props_map: &mut HashMap<*mut c_void, FnProperties>, c opcode == YARVINSN_opt_invokebuiltin_delegate_leave as i32 { // The first operand is the builtin function pointer let bf_value = *pc.add(1); - let bf_ptr = bf_value.as_ptr() as *const rb_builtin_function; + let bf_ptr: *const rb_builtin_function = bf_value.as_ptr(); if func_ptr.is_null() { func_ptr = (*bf_ptr).func_ptr as *mut c_void; @@ -140,11 +163,29 @@ pub fn init() -> Annotations { let builtin_funcs = &mut HashMap::new(); macro_rules! annotate { - ($module:ident, $method_name:literal, $return_type:expr, $($properties:ident),+) => { - let mut props = FnProperties { no_gc: false, leaf: false, elidable: false, return_type: $return_type }; + ($module:ident, $method_name:literal, $inline:ident) => { + let mut props = FnProperties::default(); + props.inline = $inline; + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; + $( + props.$properties = true; + )* + #[allow(unused_unsafe)] + annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; $( props.$properties = true; - )+ + )* + #[allow(unused_unsafe)] annotate_c_method(cfuncs, unsafe { $module }, $method_name, props); } } @@ -153,33 +194,847 @@ pub fn init() -> Annotations { ($module:ident, $method_name:literal, $return_type:expr) => { annotate_builtin!($module, $method_name, $return_type, no_gc, leaf, elidable) }; - ($module:ident, $method_name:literal, $return_type:expr, $($properties:ident),+) => { - let mut props = FnProperties { - no_gc: false, - leaf: false, - elidable: false, - return_type: $return_type - }; + ($module:ident, $method_name:literal, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + $(props.$properties = true;)+ + annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); + }; + ($module:ident, $method_name:literal, $inline:ident, $return_type:expr $(, $properties:ident)*) => { + let mut props = FnProperties::default(); + props.return_type = $return_type; + props.inline = $inline; $(props.$properties = true;)+ annotate_builtin_method(builtin_funcs, unsafe { $module }, $method_name, props); } } - annotate!(rb_mKernel, "itself", types::BasicObject, no_gc, leaf, elidable); - annotate!(rb_cString, "bytesize", types::Fixnum, no_gc, leaf); + annotate!(rb_mKernel, "itself", inline_kernel_itself); + annotate!(rb_mKernel, "block_given?", inline_kernel_block_given_p); + annotate!(rb_mKernel, "===", inline_eqq); + annotate!(rb_mKernel, "is_a?", inline_kernel_is_a_p); + annotate!(rb_cString, "bytesize", inline_string_bytesize); + annotate!(rb_cString, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "length", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cString, "getbyte", inline_string_getbyte); + annotate!(rb_cString, "setbyte", inline_string_setbyte); + annotate!(rb_cString, "empty?", inline_string_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cString, "<<", inline_string_append); + annotate!(rb_cString, "==", inline_string_eq); + // Not elidable; has a side effect of setting the encoding if ENC_CODERANGE_UNKNOWN. + // TOOD(max): Turn this into a load/compare. Will need to side-exit or do the full call if + // ENC_CODERANGE_UNKNOWN. + annotate!(rb_cString, "ascii_only?", types::BoolExact, no_gc, leaf); annotate!(rb_cModule, "name", types::StringExact.union(types::NilClass), no_gc, leaf, elidable); - annotate!(rb_cModule, "===", types::BoolExact, no_gc, leaf); - annotate!(rb_cArray, "length", types::Fixnum, no_gc, leaf, elidable); - annotate!(rb_cArray, "size", types::Fixnum, no_gc, leaf, elidable); - annotate!(rb_cNilClass, "nil?", types::TrueClass, no_gc, leaf, elidable); - annotate!(rb_mKernel, "nil?", types::FalseClass, no_gc, leaf, elidable); + annotate!(rb_cModule, "===", inline_module_eqq, types::BoolExact, no_gc, leaf); + annotate!(rb_cArray, "length", inline_array_length, types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cArray, "empty?", inline_array_empty_p, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cArray, "reverse", types::ArrayExact, leaf, elidable); + annotate!(rb_cArray, "join", types::StringExact); + annotate!(rb_cArray, "[]", inline_array_aref); + annotate!(rb_cArray, "[]=", inline_array_aset); + annotate!(rb_cArray, "<<", inline_array_push); + annotate!(rb_cArray, "push", inline_array_push); + annotate!(rb_cArray, "pop", inline_array_pop); + annotate!(rb_cHash, "[]", inline_hash_aref); + annotate!(rb_cHash, "[]=", inline_hash_aset); + annotate!(rb_cHash, "size", types::Fixnum, no_gc, leaf, elidable); + annotate!(rb_cHash, "empty?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cNilClass, "nil?", inline_nilclass_nil_p); + annotate!(rb_mKernel, "nil?", inline_kernel_nil_p); + annotate!(rb_mKernel, "respond_to?", inline_kernel_respond_to_p); + annotate!(rb_cBasicObject, "==", inline_basic_object_eq, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!", inline_basic_object_not, types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cBasicObject, "!=", inline_basic_object_neq, types::BoolExact); + annotate!(rb_cBasicObject, "initialize", inline_basic_object_initialize); + annotate!(rb_cClass, "allocate", inline_class_allocate); + annotate!(rb_cInteger, "succ", inline_integer_succ); + annotate!(rb_cInteger, "^", inline_integer_xor); + annotate!(rb_cInteger, "==", inline_integer_eq); + annotate!(rb_cInteger, "+", inline_integer_plus); + annotate!(rb_cInteger, "-", inline_integer_minus); + annotate!(rb_cInteger, "*", inline_integer_mult); + annotate!(rb_cInteger, "/", inline_integer_div); + annotate!(rb_cInteger, "%", inline_integer_mod); + annotate!(rb_cInteger, "&", inline_integer_and); + annotate!(rb_cInteger, "|", inline_integer_or); + annotate!(rb_cInteger, ">", inline_integer_gt); + annotate!(rb_cInteger, ">=", inline_integer_ge); + annotate!(rb_cInteger, "<", inline_integer_lt); + annotate!(rb_cInteger, "<=", inline_integer_le); + annotate!(rb_cInteger, "<<", inline_integer_lshift); + annotate!(rb_cInteger, ">>", inline_integer_rshift); + annotate!(rb_cInteger, "[]", inline_integer_aref); + annotate!(rb_cInteger, "to_s", types::StringExact); + annotate!(rb_cFloat, "+", inline_float_plus); + annotate!(rb_cFloat, "-", inline_float_minus); + annotate!(rb_cFloat, "*", inline_float_mul); + annotate!(rb_cFloat, "/", inline_float_div); + annotate!(rb_cFloat, "to_i", inline_float_to_i); + annotate!(rb_cFloat, "to_int", inline_float_to_i); + annotate!(rb_cString, "to_s", inline_string_to_s, types::StringExact); + annotate!(rb_cFloat, "nan?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cFloat, "finite?", types::BoolExact, no_gc, leaf, elidable); + annotate!(rb_cFloat, "infinite?", types::Fixnum.union(types::NilClass), no_gc, leaf, elidable); + let thread_singleton = unsafe { rb_singleton_class(rb_cThread) }; + annotate!(thread_singleton, "current", inline_thread_current, types::BasicObject, no_gc, leaf); + annotate_builtin!(rb_cInteger, "zero?", types::BoolExact); + annotate_builtin!(rb_cInteger, "even?", types::BoolExact); + annotate_builtin!(rb_cInteger, "odd?", types::BoolExact); + annotate_builtin!(rb_cFloat, "zero?", types::BoolExact); + annotate_builtin!(rb_cFloat, "positive?", types::BoolExact); + annotate_builtin!(rb_cFloat, "negative?", types::BoolExact); annotate_builtin!(rb_mKernel, "Float", types::Float); annotate_builtin!(rb_mKernel, "Integer", types::Integer); - annotate_builtin!(rb_mKernel, "class", types::Class, leaf); + annotate_builtin!(rb_mKernel, "class", inline_kernel_class, types::Class, leaf); + annotate_builtin!(rb_mKernel, "frozen?", types::BoolExact); + annotate_builtin!(rb_cSymbol, "name", types::StringExact); + annotate_builtin!(rb_cSymbol, "to_s", types::StringExact); + + // Array iteration builtins (used in with_jit Array#each, map, select, find) + builtin_funcs.insert(rb_jit_fixnum_inc as *mut c_void, FnProperties { inline: inline_fixnum_inc, return_type: types::Fixnum, ..Default::default() }); + builtin_funcs.insert(rb_jit_ary_at as *mut c_void, FnProperties { inline: inline_ary_at, ..Default::default() }); + builtin_funcs.insert(rb_jit_ary_at_end as *mut c_void, FnProperties { inline: inline_ary_at_end, return_type: types::BoolExact, ..Default::default() }); Annotations { cfuncs: std::mem::take(cfuncs), builtin_funcs: std::mem::take(builtin_funcs), } } + +fn no_inline(_fun: &mut hir::Function, _block: hir::BlockId, _recv: hir::InsnId, _args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + None +} + +fn inline_string_to_s(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::StringExact, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + return Some(recv); + } + None +} + +fn inline_thread_current(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let ec = fun.push_insn(block, hir::Insn::LoadEC); + let thread_ptr = fun.push_insn(block, hir::Insn::LoadField { + recv: ec, + id: FieldName::thread_ptr, + offset: RUBY_OFFSET_EC_THREAD_PTR as i32, + return_type: types::CPtr, + }); + let thread_self = fun.push_insn(block, hir::Insn::LoadField { + recv: thread_ptr, + id: FieldName::SelfParam, + offset: RUBY_OFFSET_THREAD_SELF as i32, + // TODO(max): Add Thread type. But Thread.current is not guaranteed to be an exact Thread. + // You can make subclasses... + return_type: types::BasicObject, + }); + Some(thread_self) +} + +fn inline_kernel_itself(_fun: &mut hir::Function, _block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() { + // No need to coerce the receiver; that is done by the Send rewriting. + return Some(recv); + } + None +} + +fn inline_kernel_block_given_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(fun.iseq()) }; + if unsafe { rb_get_iseq_body_type(local_iseq) } == ISEQ_TYPE_METHOD { + // Get the EP of the ISeq of the containing method, or "local level", skipping over block-level EPs. + // Equivalent of GET_LEP() macro. + let level = crate::cruby::get_lvar_level(fun.iseq()); + let lep = fun.push_insn(block, hir::Insn::GetEP { level }); + Some(fun.push_insn(block, hir::Insn::IsBlockGiven { lep })) + } else { + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) + } +} + +fn inline_array_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index] = args { + if fun.likely_a(recv, types::Array, state) + && fun.likely_a(index, types::Fixnum, state) + { + let recv = fun.coerce_to(block, recv, types::Array, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, reason: SideExitReason::GuardLess, state }); + let index = fun.push_insn(block, hir::Insn::AdjustBounds { index, length }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + let result = fun.push_insn(block, hir::Insn::ArrayAref { array: recv, index }); + return Some(result); + } + } + None +} + +fn inline_array_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if let &[index, val] = args { + if fun.likely_a(recv, types::ArrayExact, state) + && fun.likely_a(index, types::Fixnum, state) + { + let recv = fun.coerce_to(block, recv, types::ArrayExact, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + fun.guard_not_frozen(block, recv, state); + fun.guard_not_shared(block, recv, state); + + // Bounds check: unbox Fixnum index and guard 0 <= idx < length. + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let length = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let index = fun.push_insn(block, hir::Insn::GuardLess { left: index, right: length, reason: SideExitReason::GuardLess, state }); + let index = fun.push_insn(block, hir::Insn::AdjustBounds { index, length }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let index = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + + let _ = fun.push_insn(block, hir::Insn::ArrayAset { array: recv, index, val }); + fun.push_insn(block, hir::Insn::WriteBarrier { recv, val }); + return Some(val); + } + } + None +} + +fn inline_array_push(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Inline only the case of `<<` or `push` when called with a single argument. + if let &[val] = args { + if !fun.likely_a(recv, types::Array, state) { return None; } + let recv = fun.coerce_to(block, recv, types::Array, state); + fun.guard_not_frozen(block, recv, state); + let _ = fun.push_insn(block, hir::Insn::ArrayPush { array: recv, val, state }); + return Some(recv); + } + None +} + +fn inline_array_pop(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + // Only inline the case of no arguments. + let &[] = args else { return None; }; + if !fun.likely_a(recv, types::Array, state) { return None; } + let recv = fun.coerce_to(block, recv, types::Array, state); + fun.guard_not_frozen(block, recv, state); + fun.guard_not_shared(block, recv, state); + Some(fun.push_insn(block, hir::Insn::ArrayPop { array: recv, state })) +} + +fn inline_hash_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let result = fun.push_insn(block, hir::Insn::HashAref { hash: recv, key, state }); + Some(result) + } else { + None + } +} + +fn inline_hash_aset(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[key, val] = args else { return None; }; + + // Only optimize exact Hash, not subclasses + if fun.likely_a(recv, types::HashExact, state) { + let recv = fun.coerce_to(block, recv, types::HashExact, state); + let _ = fun.push_insn(block, hir::Insn::HashAset { hash: recv, key, val, state }); + // Hash#[]= returns the value, not the hash + Some(val) + } else { + None + } +} + +fn inline_string_bytesize(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if args.is_empty() && fun.likely_a(recv, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + + let result = fun.push_insn(block, hir::Insn::BoxFixnum { + val: len, + state, + }); + + return Some(result); + } + None +} + +fn inline_string_getbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) { + // String#getbyte with a Fixnum is leaf and nogc; otherwise it may run arbitrary Ruby code + // when converting the index to a C integer. + let index = fun.coerce_to(block, index, types::Fixnum, state); + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + // TODO(max): Find a way to mark these guards as not needed for correctness... as in, once + // the data dependency is gone (say, the StringGetbyte is elided), they can also be elided. + // + // This is unlike most other guards. + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, reason: SideExitReason::GuardLess, state }); + let unboxed_index = fun.push_insn(block, hir::Insn::AdjustBounds { index: unboxed_index, length: len }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + let result = fun.push_insn(block, hir::Insn::StringGetbyte { string: recv, index: unboxed_index }); + return Some(result); + } + None +} + +fn inline_string_setbyte(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index, value] = args else { return None; }; + if fun.likely_a(index, types::Fixnum, state) && fun.likely_a(value, types::Fixnum, state) { + let index = fun.coerce_to(block, index, types::Fixnum, state); + let value = fun.coerce_to(block, value, types::Fixnum, state); + + let unboxed_index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let unboxed_index = fun.push_insn(block, hir::Insn::GuardLess { left: unboxed_index, right: len, reason: SideExitReason::GuardLess, state }); + let unboxed_index = fun.push_insn(block, hir::Insn::AdjustBounds { index: unboxed_index, length: len }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + use crate::hir::SideExitReason; + let _ = fun.push_insn(block, hir::Insn::GuardGreaterEq { left: unboxed_index, right: zero, reason: SideExitReason::GuardGreaterEq, state }); + // We know that all String are HeapObject, so no need to insert a GuardType(HeapObject). + fun.guard_not_frozen(block, recv, state); + let _ = fun.push_insn(block, hir::Insn::StringSetbyteFixnum { string: recv, index, value }); + // String#setbyte returns the fixnum provided as its `value` argument back to the caller. + Some(value) + } else { + None + } +} + +fn inline_string_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + let len = fun.push_insn(block, hir::Insn::LoadField { + recv, + id: FieldName::len, + offset: RUBY_OFFSET_RSTRING_LEN as i32, + return_type: types::CInt64, + }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let is_zero = fun.push_insn(block, hir::Insn::IsBitEqual { left: len, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: is_zero }); + Some(result) +} + +fn inline_string_append(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Inline only StringExact << String, which matches original type check from + // `vm_opt_ltlt`, which checks `RB_TYPE_P(obj, T_STRING)`. + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::String, state); + let _ = fun.push_insn(block, hir::Insn::StringAppend { recv, other, state }); + return Some(recv); + } + if fun.likely_a(recv, types::StringExact, state) && fun.likely_a(other, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::StringExact, state); + let other = fun.coerce_to(block, other, types::Fixnum, state); + let _ = fun.push_insn(block, hir::Insn::StringAppendCodepoint { recv, other, state }); + return Some(recv); + } + None +} + +fn try_inline_string_equal(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, other: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if fun.likely_a(recv, types::String, state) && fun.likely_a(other, types::String, state) { + let recv = fun.coerce_to(block, recv, types::String, state); + let other = fun.coerce_to(block, other, types::String, state); + let result = fun.push_insn(block, hir::Insn::StringEqual { left: recv, right: other }); + return Some(result); + } + None +} + +fn inline_string_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_string_equal(fun, block, recv, other, state) +} + +fn inline_module_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(recv, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: other, class: recv }); + return Some(result); + } + None +} + +fn inline_array_length(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let result = fun.push_insn(block, hir::Insn::BoxFixnum { val: length_cint, state }); + return Some(result); + } + None +} + +fn inline_array_empty_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Array, state) { + let recv = fun.coerce_to(block, recv, types::Array, state); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let zero = fun.push_insn(block, hir::Insn::Const { val: hir::Const::CInt64(0) }); + let result_c = fun.push_insn(block, hir::Insn::IsBitEqual { left: length_cint, right: zero }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: result_c }); + return Some(result); + } + None +} + +fn inline_integer_succ(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + if fun.likely_a(recv, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(VALUE::fixnum_from_usize(1)) }); + let result = fun.push_insn(block, hir::Insn::FixnumAdd { left, right, state }); + return Some(result); + } + None +} + +fn inline_integer_xor(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[right] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + let left = fun.coerce_to(block, recv, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumXor { left, right }); + return Some(result); + } + None +} + +fn try_inline_float_op(fun: &mut hir::Function, block: hir::BlockId, f: &dyn Fn(hir::InsnId, hir::InsnId) -> hir::Insn, bop: u32, recv: hir::InsnId, other: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, FLOAT_REDEFINED_OP_FLAG) } { + return None; + } + // Receiver must be Flonum (cheap tag check: (val & 3) == 2). + // The other operand can be Flonum or Fixnum since rb_float_plus/minus/mul/div + // handle both via fast paths (FIXNUM_P check + cast to double). + // HeapFloat falls back to CCallWithFrame via the default Send path. + if fun.likely_a(recv, types::Flonum, state) + && (fun.likely_a(other, types::Flonum, state) || fun.likely_a(other, types::Fixnum, state)) + { + let recv = fun.coerce_to(block, recv, types::Flonum, state); + let other_type = if fun.likely_a(other, types::Flonum, state) { types::Flonum } else { types::Fixnum }; + let other = fun.coerce_to(block, other, other_type, state); + return Some(fun.push_insn(block, f(recv, other))); + } + None +} + +fn inline_float_plus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatAdd { recv, other, state }, BOP_PLUS, recv, other, state) +} + +fn inline_float_minus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatSub { recv, other, state }, BOP_MINUS, recv, other, state) +} + +fn inline_float_mul(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatMul { recv, other, state }, BOP_MULT, recv, other, state) +} + +fn inline_float_div(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_float_op(fun, block, &|recv, other| hir::Insn::FloatDiv { recv, other, state }, BOP_DIV, recv, other, state) +} + +fn inline_float_to_i(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.likely_a(recv, types::Flonum, state) { + let recv = fun.coerce_to(block, recv, types::Flonum, state); + return Some(fun.push_insn(block, hir::Insn::FloatToInt { recv, state })); + } + None +} + +fn try_inline_fixnum_op(fun: &mut hir::Function, block: hir::BlockId, f: &dyn Fn(hir::InsnId, hir::InsnId) -> hir::Insn, bop: u32, left: hir::InsnId, right: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, INTEGER_REDEFINED_OP_FLAG) } { + // If the basic operation is already redefined, we cannot optimize it. + return None; + } + if fun.likely_a(left, types::Fixnum, state) && fun.likely_a(right, types::Fixnum, state) { + if bop == BOP_NEQ { + // For opt_neq, the interpreter checks that both neq and eq are unchanged. + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ }, state }); + } + // Rely on the MethodRedefined PatchPoint for other bops. + let left = fun.coerce_to(block, left, types::Fixnum, state); + let right = fun.coerce_to(block, right, types::Fixnum, state); + return Some(fun.push_insn(block, f(left, right))); + } + None +} + +fn inline_integer_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumEq { left, right }, BOP_EQ, recv, other, state) +} + +fn inline_integer_plus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAdd { left, right, state }, BOP_PLUS, recv, other, state) +} + +fn inline_integer_minus(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumSub { left, right, state }, BOP_MINUS, recv, other, state) +} + +fn inline_integer_mult(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMult { left, right, state }, BOP_MULT, recv, other, state) +} + +fn inline_integer_div(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumDiv { left, right, state }, BOP_DIV, recv, other, state) +} + +fn inline_integer_mod(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumMod { left, right, state }, BOP_MOD, recv, other, state) +} + +fn inline_integer_and(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumAnd { left, right, }, BOP_AND, recv, other, state) +} + +fn inline_integer_or(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumOr { left, right, }, BOP_OR, recv, other, state) +} + +fn inline_integer_gt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGt { left, right }, BOP_GT, recv, other, state) +} + +fn inline_integer_ge(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumGe { left, right }, BOP_GE, recv, other, state) +} + +fn inline_integer_lt(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLt { left, right }, BOP_LT, recv, other, state) +} + +fn inline_integer_le(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLe { left, right }, BOP_LE, recv, other, state) +} + +fn inline_integer_lshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumLShift { left, right, state }, BOP_LTLT, recv, other, state) +} + +fn inline_integer_rshift(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + // Only convert to FixnumLShift if we know the shift amount is known at compile-time and could + // plausibly create a fixnum. + let Some(other_value) = fun.type_of(other).fixnum_value() else { return None; }; + // TODO(max): If other_value > 63, rewrite to constant zero. + if other_value < 0 || other_value > 63 { return None; } + try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumRShift { left, right }, BOP_GTGT, recv, other, state) +} + +fn inline_integer_aref(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[index] = args else { return None; }; + if fun.likely_a(recv, types::Fixnum, state) && fun.likely_a(index, types::Fixnum, state) { + let recv = fun.coerce_to(block, recv, types::Fixnum, state); + let index = fun.coerce_to(block, index, types::Fixnum, state); + let result = fun.push_insn(block, hir::Insn::FixnumAref { recv, index }); + return Some(result); + } + None +} + +fn inline_basic_object_eq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_basic_object_not(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[] = args else { return None; }; + if fun.type_of(recv).is_known_truthy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) }); + return Some(result); + } + if fun.type_of(recv).is_known_falsy() { + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) }); + return Some(result); + } + None +} + +fn try_inline_string_not_equal(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, other: hir::InsnId, state: hir::InsnId) -> Option<hir::InsnId> { + if !fun.likely_a(recv, types::String, state) || !fun.likely_a(other, types::String, state) { + return None; + } + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + + // String#!= is lowered to #==. Keep this specialization only while #== + // still resolves to rb_str_equal. + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_str_equal as _, state) { + return None; + } + + let eq_result = try_inline_string_equal(fun, block, recv, other, state)?; + // StringEqual always returns a Ruby boolean (Qtrue/Qfalse), + // so `!=` can be lowered to `eq_result != Qtrue`. + let true_val = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) }); + let not_equal = fun.push_insn(block, hir::Insn::IsBitNotEqual { left: eq_result, right: true_val }); + Some(fun.push_insn(block, hir::Insn::BoxBool { val: not_equal })) +} + +fn inline_basic_object_neq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if let Some(result) = try_inline_fixnum_op(fun, block, &|left, right| hir::Insn::FixnumNeq { left, right }, BOP_NEQ, recv, other, state) { + return Some(result); + } + + if let Some(result) = try_inline_string_not_equal(fun, block, recv, other, state) { + return Some(result); + } + + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitNotEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_class_allocate(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + + // Inline only in the case we have a leaf allocator + fun.try_inline_object_alloc(block, recv, state) +} + +fn inline_basic_object_initialize(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + let result = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qnil) }); + Some(result) +} + +fn inline_nilclass_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qtrue) })) +} + +fn inline_eqq(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + if !fun.assume_expected_cfunc(block, recv_class, ID!(eq), rb_obj_equal as _, state) { + return None; + } + let c_result = fun.push_insn(block, hir::Insn::IsBitEqual { left: recv, right: other }); + let result = fun.push_insn(block, hir::Insn::BoxBool { val: c_result }); + Some(result) +} + +fn inline_kernel_is_a_p(fun: &mut hir::Function, block: hir::BlockId, recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[other] = args else { return None; }; + if fun.is_a(other, types::Class) { + let result = fun.push_insn(block, hir::Insn::IsA { val: recv, class: other }); + return Some(result); + } + None +} + +fn inline_kernel_nil_p(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + if !args.is_empty() { return None; } + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(Qfalse) })) +} + +fn inline_kernel_respond_to_p( + fun: &mut hir::Function, + block: hir::BlockId, + recv: hir::InsnId, + args: &[hir::InsnId], + state: hir::InsnId, +) -> Option<hir::InsnId> { + // Parse arguments: respond_to?(method_name, allow_priv = false) + let (method_name, allow_priv) = match *args { + [method_name] => (method_name, false), + [method_name, arg] => match fun.type_of(arg) { + t if t.is_known_truthy() => (method_name, true), + t if t.is_known_falsy() => (method_name, false), + // Unknown type; bail out + _ => return None, + }, + // Unknown args; bail out + _ => return None, + }; + + // Method name must be a static symbol + let method_name = fun.type_of(method_name).ruby_object()?; + if !method_name.static_sym_p() { + return None; + } + + // The receiver must have a known class to call `respond_to?` on + // TODO: This is technically overly strict. This would also work if all of the + // observed objects at this point agree on `respond_to?` and we can add many patchpoints. + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + + // Get the method ID and its corresponding callable method entry + let mid = unsafe { rb_sym2id(method_name) }; + let target_cme = unsafe { rb_callable_method_entry_or_negative(recv_class, mid) }; + assert!( + !target_cme.is_null(), + "Should never be null, as in that case we will be returned a \"negative CME\"" + ); + + let cme_def_type = unsafe { get_cme_def_type(target_cme) }; + + // Cannot inline a refined method, since their refinement depends on lexical scope + if cme_def_type == VM_METHOD_TYPE_REFINED { + return None; + } + + let visibility = match cme_def_type { + VM_METHOD_TYPE_UNDEF => METHOD_VISI_UNDEF, + _ => unsafe { METHOD_ENTRY_VISI(target_cme) }, + }; + + let result = match (visibility, allow_priv) { + // Method undefined; check `respond_to_missing?` + (METHOD_VISI_UNDEF, _) => { + let respond_to_missing = ID!(respond_to_missing); + if unsafe { rb_method_basic_definition_p(recv_class, respond_to_missing) } == 0 { + return None; // Custom definition of respond_to_missing?, so cannot inline + } + let respond_to_missing_cme = + unsafe { rb_callable_method_entry(recv_class, respond_to_missing) }; + // Protect against redefinition of `respond_to_missing?` + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::NoTracePoint, + state, + }, + ); + fun.push_insn( + block, + hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: respond_to_missing, + cme: respond_to_missing_cme, + }, + state, + }, + ); + Qfalse + } + // Private method with allow priv=false, so `respond_to?` returns false + (METHOD_VISI_PRIVATE, false) => Qfalse, + // Public method or allow_priv=true: check if implemented + (METHOD_VISI_PUBLIC, _) | (_, true) => { + if cme_def_type == VM_METHOD_TYPE_NOTIMPLEMENTED { + // C method with rb_f_notimplement(). `respond_to?` returns false + // without consulting `respond_to_missing?`. See also: rb_add_method_cfunc() + Qfalse + } else { + Qtrue + } + } + (_, _) => return None, // not public and include_all not known, can't compile + }; + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + return None; + } + fun.push_insn(block, hir::Insn::PatchPoint { invariant: hir::Invariant::NoTracePoint, state }); + fun.push_insn(block, hir::Insn::PatchPoint { + invariant: hir::Invariant::MethodRedefined { + klass: recv_class, + method: mid, + cme: target_cme + }, state + }); + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(result) })) +} + +fn inline_kernel_class(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv] = args else { return None; }; + let recv_class = fun.type_of(recv).runtime_exact_ruby_class()?; + let real_class = unsafe { rb_class_real(recv_class) }; + Some(fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(real_class) })) +} + +/// Inline `fixnum_inc(ec, self, num)` implies FixnumAdd(num, 1). +/// num is always a Fixnum (starts at 0 and is incremented by fixnum_inc). +fn inline_fixnum_inc(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[_self, num] = args else { return None; }; + let one = fun.push_insn(block, hir::Insn::Const { val: hir::Const::Value(VALUE::fixnum_from_usize(1)) }); + let result = fun.push_insn(block, hir::Insn::FixnumAdd { left: num, right: one, state }); + Some(result) +} + +/// Inline `ary_at(ec, self, index)` implies ArrayAref. +/// Called from Array#each etc. where self is Array and index is in bounds. +fn inline_ary_at(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], _state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv, index] = args else { return None; }; + let recv = fun.push_insn(block, hir::Insn::RefineType { val: recv, new_type: types::Array }); + let index = fun.push_insn(block, hir::Insn::UnboxFixnum { val: index }); + let result = fun.push_insn(block, hir::Insn::ArrayAref { array: recv, index }); + Some(result) +} + +/// Inline `ary_at_end(ec, self, index)` implies index >= ArrayLength(self). +/// Called from Array#each etc. where self is Array and index is Fixnum. +fn inline_ary_at_end(fun: &mut hir::Function, block: hir::BlockId, _recv: hir::InsnId, args: &[hir::InsnId], state: hir::InsnId) -> Option<hir::InsnId> { + let &[recv, index] = args else { return None; }; + let recv = fun.push_insn(block, hir::Insn::RefineType { val: recv, new_type: types::Array }); + let length_cint = fun.push_insn(block, hir::Insn::ArrayLength { array: recv }); + let length = fun.push_insn(block, hir::Insn::BoxFixnum { val: length_cint, state }); + let result = fun.push_insn(block, hir::Insn::FixnumGe { left: index, right: length }); + Some(result) +} diff --git a/zjit/src/disasm.rs b/zjit/src/disasm.rs index 09864ef649..36bb90cff7 100644 --- a/zjit/src/disasm.rs +++ b/zjit/src/disasm.rs @@ -1,7 +1,24 @@ -use crate::asm::CodeBlock; +use crate::{asm::CodeBlock, options::DumpDisasm, virtualmem::CodePtr}; -pub const BOLD_BEGIN: &str = "\x1b[1m"; -pub const BOLD_END: &str = "\x1b[22m"; +/// Dump disassembly for a range in a [CodeBlock]. +pub fn dump_disasm_addr_range(cb: &CodeBlock, start_addr: CodePtr, end_addr: CodePtr, dump_disasm: &DumpDisasm) { + let disasm = disasm_addr_range(cb, start_addr.raw_ptr(cb) as usize, end_addr.raw_ptr(cb) as usize); + if disasm.is_empty() { + return; + } + + match dump_disasm { + DumpDisasm::Stdout => println!("{disasm}"), + DumpDisasm::File(fd) => { + use std::io::Write; + use std::os::unix::io::{FromRawFd, IntoRawFd}; + + let mut file = unsafe { std::fs::File::from_raw_fd(*fd) }; + file.write_all(disasm.as_bytes()).unwrap(); + let _ = file.into_raw_fd(); + } + } +} pub fn disasm_addr_range(cb: &CodeBlock, start_addr: usize, end_addr: usize) -> String { use std::fmt::Write; @@ -36,16 +53,20 @@ pub fn disasm_addr_range(cb: &CodeBlock, start_addr: usize, end_addr: usize) -> let start_addr = 0; let insns = cs.disasm_all(code_slice, start_addr as u64).unwrap(); + let colors = crate::ttycolors::get_colors(); + let bold_begin = colors.bold_begin; + let bold_end = colors.bold_end; + // For each instruction in this block for insn in insns.as_ref() { // Comments for this block if let Some(comment_list) = cb.comments_at(insn.address() as usize) { for comment in comment_list { - writeln!(&mut out, " {BOLD_BEGIN}# {comment}{BOLD_END}").unwrap(); + writeln!(&mut out, " {bold_begin}# {comment}{bold_end}").unwrap(); } } writeln!(&mut out, " {}", format!("{insn}").trim()).unwrap(); } - return out; + out } diff --git a/zjit/src/distribution.rs b/zjit/src/distribution.rs index 5927ffa5c9..aa4667b939 100644 --- a/zjit/src/distribution.rs +++ b/zjit/src/distribution.rs @@ -1,14 +1,18 @@ +//! Type frequency distribution tracker. + +use crate::options::NumProfiles; + /// This implementation was inspired by the type feedback module from Google's S6, which was /// written in C++ for use with Python. This is a new implementation in Rust created for use with /// Ruby instead of Python. #[derive(Debug, Clone)] pub struct Distribution<T: Copy + PartialEq + Default, const N: usize> { /// buckets and counts have the same length - /// buckets[0] is always the most common item + /// `buckets[0]` is always the most common item buckets: [T; N], - counts: [usize; N], + counts: [NumProfiles; N], /// if there is no more room, increment the fallback - other: usize, + other: NumProfiles, // TODO(max): Add count disparity, which can help determine when to reset the distribution } @@ -21,13 +25,13 @@ impl<T: Copy + PartialEq + Default, const N: usize> Distribution<T, N> { for (bucket, count) in self.buckets.iter_mut().zip(self.counts.iter_mut()) { if *bucket == item || *count == 0 { *bucket = item; - *count += 1; + *count = count.saturating_add(1); // Keep the most frequent item at the front self.bubble_up(); return; } } - self.other += 1; + self.other = self.other.saturating_add(1); } /// Keep the highest counted bucket at index 0 @@ -67,7 +71,7 @@ enum DistributionKind { SkewedMegamorphic, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct DistributionSummary<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> { kind: DistributionKind, buckets: [T; N], @@ -85,7 +89,7 @@ impl<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> Distributi assert!(first_count >= count, "First count should be the largest"); } } - let num_seen = dist.counts.iter().sum::<usize>() + dist.other; + let num_seen = dist.counts.iter().map(|&c| usize::from(c)).sum::<usize>() + usize::from(dist.other); let kind = if dist.other == 0 { // Seen <= N types total if dist.counts[0] == 0 { @@ -105,17 +109,25 @@ impl<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> Distributi DistributionKind::Megamorphic } }; - Self { kind, buckets: dist.buckets.clone() } + Self { kind, buckets: dist.buckets } } pub fn is_monomorphic(&self) -> bool { self.kind == DistributionKind::Monomorphic } + pub fn is_polymorphic(&self) -> bool { + self.kind == DistributionKind::Polymorphic + } + pub fn is_skewed_polymorphic(&self) -> bool { self.kind == DistributionKind::SkewedPolymorphic } + pub fn is_megamorphic(&self) -> bool { + self.kind == DistributionKind::Megamorphic + } + pub fn is_skewed_megamorphic(&self) -> bool { self.kind == DistributionKind::SkewedMegamorphic } @@ -124,6 +136,10 @@ impl<T: Copy + PartialEq + Default + std::fmt::Debug, const N: usize> Distributi assert!(idx < N, "index {idx} out of bounds for buckets[{N}]"); self.buckets[idx] } + + pub fn buckets(&self) -> &[T] { + &self.buckets + } } #[cfg(test)] diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index 3462b80232..7f5bc7891f 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -1,86 +1,11 @@ -// This module is responsible for marking/moving objects on GC. +//! This module is responsible for marking/moving objects on GC. -use std::cell::RefCell; -use std::rc::Rc; +use std::ptr::null; use std::{ffi::c_void, ops::Range}; -use crate::codegen::IseqCall; -use crate::{cruby::*, profile::IseqProfile, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; +use crate::{cruby::*, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; +use crate::payload::{IseqPayload, IseqVersionRef, get_or_create_iseq_payload}; use crate::stats::Counter::gc_time_ns; -/// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. -#[derive(Debug)] -pub struct IseqPayload { - /// Compilation status of the ISEQ. It has the JIT code address of the first block if Compiled. - pub status: IseqStatus, - - /// Type information of YARV instruction operands - pub profile: IseqProfile, - - /// GC offsets of the JIT code. These are the addresses of objects that need to be marked. - pub gc_offsets: Vec<CodePtr>, - - /// JIT-to-JIT calls in the ISEQ. The IseqPayload's ISEQ is the caller of it. - pub iseq_calls: Vec<Rc<RefCell<IseqCall>>>, -} - -impl IseqPayload { - fn new(iseq_size: u32) -> Self { - Self { - status: IseqStatus::NotCompiled, - profile: IseqProfile::new(iseq_size), - gc_offsets: vec![], - iseq_calls: vec![], - } - } -} - -#[derive(Debug, PartialEq)] -pub enum IseqStatus { - /// CodePtr has the JIT code address of the first block - Compiled(CodePtr), - CantCompile, - NotCompiled, -} - -/// Get a pointer to the payload object associated with an ISEQ. Create one if none exists. -pub fn get_or_create_iseq_payload_ptr(iseq: IseqPtr) -> *mut IseqPayload { - type VoidPtr = *mut c_void; - - unsafe { - let payload = rb_iseq_get_zjit_payload(iseq); - if payload.is_null() { - // Allocate a new payload with Box and transfer ownership to the GC. - // We drop the payload with Box::from_raw when the GC frees the ISEQ and calls us. - // NOTE(alan): Sometimes we read from an ISEQ without ever writing to it. - // We allocate in those cases anyways. - let iseq_size = get_iseq_encoded_size(iseq); - let new_payload = IseqPayload::new(iseq_size); - let new_payload = Box::into_raw(Box::new(new_payload)); - rb_iseq_set_zjit_payload(iseq, new_payload as VoidPtr); - - new_payload - } else { - payload as *mut IseqPayload - } - } -} - -/// Get the payload object associated with an ISEQ. Create one if none exists. -pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { - let payload_non_null = get_or_create_iseq_payload_ptr(iseq); - payload_ptr_as_mut(payload_non_null) -} - -/// Convert an IseqPayload pointer to a mutable reference. Only one reference -/// should be kept at a time. -fn payload_ptr_as_mut(payload_ptr: *mut IseqPayload) -> &'static mut IseqPayload { - // SAFETY: we should have the VM lock and all other Ruby threads should be asleep. So we have - // exclusive mutable access. - // Hmm, nothing seems to stop calling this on the same - // iseq twice, though, which violates aliasing rules. - unsafe { payload_ptr.as_mut() }.unwrap() -} - /// GC callback for marking GC objects in the per-ISEQ payload. #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_iseq_mark(payload: *mut c_void) { @@ -119,6 +44,43 @@ pub extern "C" fn rb_zjit_iseq_update_references(payload: *mut c_void) { with_time_stat(gc_time_ns, || iseq_update_references(payload)); } +/// GC callback for finalizing an ISEQ +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_iseq_free(iseq: IseqPtr) { + if !ZJITState::has_instance() { + return; + } + + // TODO(Shopify/ruby#682): Free `IseqPayload` + let payload = get_or_create_iseq_payload(iseq); + for version in payload.versions.iter_mut() { + unsafe { version.as_mut() }.iseq = null(); + } + + let invariants = ZJITState::get_invariants(); + invariants.forget_iseq(iseq); +} + +/// GC callback for finalizing a CME +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_cme_free(cme: *const rb_callable_method_entry_struct) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_cme(cme); +} + +/// GC callback for finalizing a class +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_klass_free(klass: VALUE) { + if !ZJITState::has_instance() { + return; + } + let invariants = ZJITState::get_invariants(); + invariants.forget_klass(klass); +} + /// GC callback for updating object references after all object moves #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_root_update_references() { @@ -127,6 +89,14 @@ pub extern "C" fn rb_zjit_root_update_references() { } let invariants = ZJITState::get_invariants(); invariants.update_references(); + + // Update iseq pointers in all JITFrames for GC compaction. + // rb_execution_context_update only updates JITFrames currently on the stack, + // but JITFrames not on the stack also need their iseq pointers updated + // because the JIT code will reuse them on the next call. + for &jit_frame in ZJITState::get_jit_frames().iter() { + unsafe { &mut *jit_frame }.update_references(); + } } fn iseq_mark(payload: &IseqPayload) { @@ -137,14 +107,16 @@ fn iseq_mark(payload: &IseqPayload) { // Mark objects baked in JIT code let cb = ZJITState::get_code_block(); - for &offset in payload.gc_offsets.iter() { - let value_ptr: *const u8 = offset.raw_ptr(cb); - // Creating an unaligned pointer is well defined unlike in C. - let value_ptr = value_ptr as *const VALUE; - - unsafe { - let object = value_ptr.read_unaligned(); - rb_gc_mark_movable(object); + for version in payload.versions.iter() { + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { + let value_ptr: *const u8 = offset.raw_ptr(cb); + // Creating an unaligned pointer is well defined unlike in C. + let value_ptr = value_ptr as *const VALUE; + + unsafe { + let object = value_ptr.read_unaligned(); + rb_gc_mark_movable(object); + } } } } @@ -159,18 +131,38 @@ fn iseq_update_references(payload: &mut IseqPayload) { } }); - // Move ISEQ references in IseqCall - for iseq_call in payload.iseq_calls.iter_mut() { - let old_iseq = iseq_call.borrow().iseq; + for &version in payload.versions.iter() { + iseq_version_update_references(version); + } +} + +fn iseq_version_update_references(mut version: IseqVersionRef) { + // Move ISEQ in the payload + unsafe { version.as_mut() }.iseq = unsafe { rb_gc_location(version.as_ref().iseq.into()) }.as_iseq(); + + // Move ISEQ references in incoming IseqCalls + for iseq_call in unsafe { version.as_mut() }.incoming.iter_mut() { + let old_iseq = iseq_call.iseq.get(); + let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; + if old_iseq != new_iseq { + iseq_call.iseq.set(new_iseq); + } + } + + // Move ISEQ references in outgoing IseqCalls + for iseq_call in unsafe { version.as_mut() }.outgoing.iter_mut() { + let old_iseq = iseq_call.iseq.get(); let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; if old_iseq != new_iseq { - iseq_call.borrow_mut().iseq = new_iseq; + iseq_call.iseq.set(new_iseq); } } - // Move objects baked in JIT code + // Move objects baked in JIT code. + // The code region is already writable because rb_zjit_mark_all_writable() was called + // before the GC update_references phase. We write directly to avoid per-page mprotect calls. let cb = ZJITState::get_code_block(); - for &offset in payload.gc_offsets.iter() { + for &offset in unsafe { version.as_ref() }.gc_offsets.iter() { let value_ptr: *const u8 = offset.raw_ptr(cb); // Creating an unaligned pointer is well defined unlike in C. let value_ptr = value_ptr as *const VALUE; @@ -180,19 +172,15 @@ fn iseq_update_references(payload: &mut IseqPayload) { // Only write when the VALUE moves, to be copy-on-write friendly. if new_addr != object { - for (byte_idx, &byte) in new_addr.as_u64().to_le_bytes().iter().enumerate() { - let byte_code_ptr = offset.add_bytes(byte_idx); - cb.write_mem(byte_code_ptr, byte).expect("patching existing code should be within bounds"); - } + let value_ptr = value_ptr as *mut VALUE; + unsafe { value_ptr.write_unaligned(new_addr) }; } } - cb.mark_all_executable(); } /// Append a set of gc_offsets to the iseq's payload -pub fn append_gc_offsets(iseq: IseqPtr, offsets: &Vec<CodePtr>) { - let payload = get_or_create_iseq_payload(iseq); - payload.gc_offsets.extend(offsets); +pub fn append_gc_offsets(iseq: IseqPtr, mut version: IseqVersionRef, offsets: &Vec<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.extend(offsets); // Call writebarrier on each newly added value let cb = ZJITState::get_code_block(); @@ -201,7 +189,7 @@ pub fn append_gc_offsets(iseq: IseqPtr, offsets: &Vec<CodePtr>) { let value_ptr = value_ptr as *const VALUE; unsafe { let object = value_ptr.read_unaligned(); - rb_gc_writebarrier(iseq.into(), object); + VALUE::from(iseq).write_barrier(object); } } } @@ -210,15 +198,47 @@ pub fn append_gc_offsets(iseq: IseqPtr, offsets: &Vec<CodePtr>) { /// We do this when invalidation rewrites some code with a jump instruction /// and GC offsets are corrupted by the rewrite, assuming no on-stack code /// will step into the instruction with the GC offsets after invalidation. -pub fn remove_gc_offsets(payload_ptr: *mut IseqPayload, removed_range: &Range<CodePtr>) { - let payload = payload_ptr_as_mut(payload_ptr); - payload.gc_offsets.retain(|&gc_offset| { +pub fn remove_gc_offsets(mut version: IseqVersionRef, removed_range: &Range<CodePtr>) { + unsafe { version.as_mut() }.gc_offsets.retain(|&gc_offset| { let offset_range = gc_offset..(gc_offset.add_bytes(SIZEOF_VALUE)); !ranges_overlap(&offset_range, removed_range) }); } -/// Return true if given Range<CodePtr> ranges overlap with each other +/// Return true if given `Range<CodePtr>` ranges overlap with each other fn ranges_overlap<T>(left: &Range<T>, right: &Range<T>) -> bool where T: PartialOrd { left.start < right.end && right.start < left.end } + +/// GC callback for making all JIT code writable before updating references in bulk. +/// This avoids toggling W^X permissions per-page during GC compaction. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_mark_all_writable() { + if !ZJITState::has_instance() { + return; + } + ZJITState::get_code_block().mark_all_writable(); +} + +/// GC callback for making all JIT code executable after updating references in bulk. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_mark_all_executable() { + if !ZJITState::has_instance() { + return; + } + ZJITState::get_code_block().mark_all_executable(); +} + +/// Callback for marking GC objects inside [crate::invariants::Invariants]. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_root_mark() { + // Mark iseq pointers in all JITFrames. JITFrames that are currently on the + // stack are also marked via rb_execution_context_mark, but JITFrames not on + // the stack still need their iseqs kept alive because JIT code will reuse them. + if !ZJITState::has_instance() { + return; + } + for &jit_frame in ZJITState::get_jit_frames().iter() { + unsafe { &*jit_frame }.mark(); + } +} diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 1cd31497d8..fa035292e4 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -3,15 +3,46 @@ // We use the YARV bytecode constants which have a CRuby-style name #![allow(non_upper_case_globals)] +#![allow(clippy::if_same_then_else)] +#![allow(clippy::match_like_matches_macro)] use crate::{ - cast::IntoUsize, cruby::*, gc::{get_or_create_iseq_payload, IseqPayload}, options::{get_option, DumpHIR}, state::ZJITState, stats::Counter + backend::lir::C_ARG_OPNDS, + cast::IntoUsize, codegen::{local_idx_to_ep_offset, max_iseq_versions}, cruby::*, invariants::{self}, payload::{get_or_create_iseq_payload, IseqPayload}, options::{debug, get_option, DumpHIR}, state::ZJITState, json::Json, + state, }; use std::{ - cell::RefCell, collections::{HashMap, HashSet, VecDeque}, ffi::{c_int, c_void, CStr}, fmt::Display, mem::{align_of, size_of}, ptr, slice::Iter + cell::RefCell, collections::{HashMap, HashSet, VecDeque}, ffi::{c_void, c_uint, c_int, CStr}, fmt::Display, mem::{align_of, size_of}, ptr, slice::Iter, + sync::atomic::Ordering, }; use crate::hir_type::{Type, types}; +use crate::hir_effect::{Effect, abstract_heaps, effects}; use crate::bitset::BitSet; use crate::profile::{TypeDistributionSummary, ProfiledType}; +use crate::stats::Counter; +use SendFallbackReason::*; + +pub(crate) mod tests; +mod opt_tests; + +#[allow(unused_macros)] +macro_rules! hir_comment { + ($func:expr, $block:expr, $($arg:tt)*) => { + // If a diagnostic dump is requested, enrich it with HIR comments. Otherwise, avoid + // allocating comment strings or adding comment instructions that nobody can observe. + let enable_comment = $crate::options::get_option_ref!(dump_hir_init).is_some() || + $crate::options::get_option_ref!(dump_hir_opt).is_some() || + $crate::options::get_option_ref!(dump_hir_graphviz).is_some() || + $crate::options::get_option!(dump_hir_iongraph) || + $crate::options::get_option_ref!(dump_lir).is_some() || + $crate::options::get_option_ref!(dump_disasm).is_some(); + if enable_comment { + $func.push_comment($block, format!($($arg)*)); + } + }; +} + +#[allow(unused_imports)] +pub(crate) use hir_comment; /// An index of an [`Insn`] in a [`Function`]. This is a popular /// type since this effectively acts as a pointer to an [`Insn`]. @@ -19,9 +50,9 @@ use crate::profile::{TypeDistributionSummary, ProfiledType}; #[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash, Debug)] pub struct InsnId(pub usize); -impl Into<usize> for InsnId { - fn into(self) -> usize { - self.0 +impl From<InsnId> for usize { + fn from(val: InsnId) -> Self { + val.0 } } @@ -32,12 +63,12 @@ impl std::fmt::Display for InsnId { } /// The index of a [`Block`], which effectively acts like a pointer. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, PartialOrd, Ord)] pub struct BlockId(pub usize); -impl Into<usize> for BlockId { - fn into(self) -> usize { - self.0 +impl From<BlockId> for usize { + fn from(val: BlockId) -> Self { + val.0 } } @@ -109,7 +140,7 @@ impl std::fmt::Display for BranchEdge { } /// Invalidation reasons -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Invariant { /// Basic operation is redefined BOPRedefined { @@ -131,8 +162,20 @@ pub enum Invariant { StableConstantNames { idlist: *const ID, }, + /// TracePoint is not enabled. If TracePoint is enabled, this is invalidated. + NoTracePoint, + /// cfp->ep is not escaped to the heap on the ISEQ + NoEPEscape(IseqPtr), /// There is one ractor running. If a non-root ractor gets spawned, this is invalidated. SingleRactorMode, + /// Objects of this class have no singleton class. + /// When a singleton class is created for an object of this class, this is invalidated. + NoSingletonClass { + klass: VALUE, + }, + /// Only the root box is active, so we can safely read from the prime classext. + /// Invalidated if a non-root box duplicates any classext. + RootBoxOnly, } impl Invariant { @@ -160,7 +203,7 @@ impl From<u32> for SpecialObjectType { VM_SPECIAL_OBJECT_VMCORE => SpecialObjectType::VMCore, VM_SPECIAL_OBJECT_CBASE => SpecialObjectType::CBase, VM_SPECIAL_OBJECT_CONST_BASE => SpecialObjectType::ConstBase, - _ => panic!("Invalid special object type: {}", value), + _ => panic!("Invalid special object type: {value}"), } } } @@ -201,21 +244,40 @@ impl<'a> std::fmt::Display for InvariantPrinter<'a> { } write!(f, ", ")?; match bop { - BOP_PLUS => write!(f, "BOP_PLUS")?, - BOP_MINUS => write!(f, "BOP_MINUS")?, - BOP_MULT => write!(f, "BOP_MULT")?, - BOP_DIV => write!(f, "BOP_DIV")?, - BOP_MOD => write!(f, "BOP_MOD")?, - BOP_EQ => write!(f, "BOP_EQ")?, - BOP_NEQ => write!(f, "BOP_NEQ")?, - BOP_LT => write!(f, "BOP_LT")?, - BOP_LE => write!(f, "BOP_LE")?, - BOP_GT => write!(f, "BOP_GT")?, - BOP_GE => write!(f, "BOP_GE")?, - BOP_FREEZE => write!(f, "BOP_FREEZE")?, - BOP_UMINUS => write!(f, "BOP_UMINUS")?, - BOP_MAX => write!(f, "BOP_MAX")?, - BOP_AREF => write!(f, "BOP_AREF")?, + BOP_PLUS => write!(f, "BOP_PLUS")?, + BOP_MINUS => write!(f, "BOP_MINUS")?, + BOP_MULT => write!(f, "BOP_MULT")?, + BOP_DIV => write!(f, "BOP_DIV")?, + BOP_MOD => write!(f, "BOP_MOD")?, + BOP_EQ => write!(f, "BOP_EQ")?, + BOP_EQQ => write!(f, "BOP_EQQ")?, + BOP_LT => write!(f, "BOP_LT")?, + BOP_LE => write!(f, "BOP_LE")?, + BOP_LTLT => write!(f, "BOP_LTLT")?, + BOP_AREF => write!(f, "BOP_AREF")?, + BOP_ASET => write!(f, "BOP_ASET")?, + BOP_LENGTH => write!(f, "BOP_LENGTH")?, + BOP_SIZE => write!(f, "BOP_SIZE")?, + BOP_EMPTY_P => write!(f, "BOP_EMPTY_P")?, + BOP_NIL_P => write!(f, "BOP_NIL_P")?, + BOP_SUCC => write!(f, "BOP_SUCC")?, + BOP_GT => write!(f, "BOP_GT")?, + BOP_GE => write!(f, "BOP_GE")?, + BOP_NOT => write!(f, "BOP_NOT")?, + BOP_NEQ => write!(f, "BOP_NEQ")?, + BOP_MATCH => write!(f, "BOP_MATCH")?, + BOP_FREEZE => write!(f, "BOP_FREEZE")?, + BOP_UMINUS => write!(f, "BOP_UMINUS")?, + BOP_MAX => write!(f, "BOP_MAX")?, + BOP_MIN => write!(f, "BOP_MIN")?, + BOP_HASH => write!(f, "BOP_HASH")?, + BOP_CALL => write!(f, "BOP_CALL")?, + BOP_AND => write!(f, "BOP_AND")?, + BOP_OR => write!(f, "BOP_OR")?, + BOP_CMP => write!(f, "BOP_CMP")?, + BOP_DEFAULT => write!(f, "BOP_DEFAULT")?, + BOP_PACK => write!(f, "BOP_PACK")?, + BOP_INCLUDE_P => write!(f, "BOP_INCLUDE_P")?, _ => write!(f, "{bop}")?, } write!(f, ")") @@ -244,12 +306,21 @@ impl<'a> std::fmt::Display for InvariantPrinter<'a> { } write!(f, ")") } + Invariant::NoTracePoint => write!(f, "NoTracePoint"), + Invariant::NoEPEscape(iseq) => write!(f, "NoEPEscape({})", &iseq_name(iseq)), Invariant::SingleRactorMode => write!(f, "SingleRactorMode"), + Invariant::NoSingletonClass { klass } => { + let class_name = get_class_name(klass); + write!(f, "NoSingletonClass({}@{:p})", + class_name, + self.ptr_map.map_ptr(klass.as_ptr::<VALUE>())) + } + Invariant::RootBoxOnly => write!(f, "RootBoxOnly"), } } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Copy)] pub enum Const { Value(VALUE), CBool(bool), @@ -260,8 +331,10 @@ pub enum Const { CUInt8(u8), CUInt16(u16), CUInt32(u32), + CAttrIndex(attr_index_t), + CShape(ShapeId), CUInt64(u64), - CPtr(*mut u8), + CPtr(*const u8), CDouble(f64), } @@ -272,11 +345,12 @@ impl std::fmt::Display for Const { } impl Const { - fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> ConstPrinter<'a> { + pub fn print<'a>(&'a self, ptr_map: &'a PtrPrintMap) -> ConstPrinter<'a> { ConstPrinter { inner: self, ptr_map } } } +#[derive(Clone, Copy)] pub enum RangeType { Inclusive = 0, // include the end value Exclusive = 1, // exclude the end value @@ -293,34 +367,20 @@ impl std::fmt::Display for RangeType { impl std::fmt::Debug for RangeType { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", self) + write!(f, "{self}") } } -impl Clone for RangeType { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for RangeType {} - impl From<u32> for RangeType { fn from(flag: u32) -> Self { match flag { 0 => RangeType::Inclusive, 1 => RangeType::Exclusive, - _ => panic!("Invalid range flag: {}", flag), + _ => panic!("Invalid range flag: {flag}"), } } } -impl From<RangeType> for u32 { - fn from(range_type: RangeType) -> Self { - range_type as u32 - } -} - /// Special regex backref symbol types #[derive(Debug, Clone, Copy, PartialEq)] pub enum SpecialBackrefSymbol { @@ -339,13 +399,13 @@ impl TryFrom<u8> for SpecialBackrefSymbol { '`' => Ok(SpecialBackrefSymbol::PreMatch), '\'' => Ok(SpecialBackrefSymbol::PostMatch), '+' => Ok(SpecialBackrefSymbol::LastGroup), - c => Err(format!("invalid backref symbol: '{}'", c)), + c => Err(format!("invalid backref symbol: '{c}'")), } } } /// Print adaptor for [`Const`]. See [`PtrPrintMap`]. -struct ConstPrinter<'a> { +pub struct ConstPrinter<'a> { inner: &'a Const, ptr_map: &'a PtrPrintMap, } @@ -354,7 +414,17 @@ impl<'a> std::fmt::Display for ConstPrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self.inner { Const::Value(val) => write!(f, "Value({})", val.print(self.ptr_map)), - Const::CPtr(val) => write!(f, "CPtr({:p})", self.ptr_map.map_ptr(val)), + // Since `&` coerces to a raw pointer, be careful to get `val` and not `&val` here. + &Const::CPtr(val) => write!(f, "CPtr({:p})", self.ptr_map.map_ptr(val)), + &Const::CShape(shape_id) => write!(f, "CShape({:p})", self.ptr_map.map_shape(shape_id)), + &Const::CUInt64(int) => { + // Print in hex if signed bit is set + if 0 != int & (1 << (u64::BITS - 1)) { + write!(f, "CUInt64(0x{int:x})") + } else { + write!(f, "CUInt64({int})") + } + } _ => write!(f, "{:?}", self.inner), } } @@ -393,9 +463,19 @@ impl PtrPrintMap { } } +struct Offset(i32); + +impl std::fmt::LowerHex for Offset { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let prefix = if f.alternate() { "0x" } else { "" }; + let bare_hex = format!("{:x}", self.0.abs()); + f.pad_integral(self.0 >= 0, prefix, &bare_hex) + } +} + impl PtrPrintMap { /// Map a pointer for printing - fn map_ptr<T>(&self, ptr: *const T) -> *const T { + pub fn map_ptr<T>(&self, ptr: *const T) -> *const T { // When testing, address stability is not a concern so print real address to enable code // reuse if !self.map_ptrs { @@ -423,62 +503,381 @@ impl PtrPrintMap { fn map_id(&self, id: u64) -> *const c_void { self.map_ptr(id as *const c_void) } + + /// Map an index into a Ruby object (e.g. for an ivar) for printing + fn map_index(&self, id: u64) -> *const c_void { + self.map_ptr(id as *const c_void) + } + + fn map_offset(&self, id: i32) -> Offset { + Offset(self.map_ptr(id as *const c_void) as i32) + } + + /// Map shape ID into a pointer for printing + pub fn map_shape(&self, id: ShapeId) -> *const c_void { + self.map_ptr(id.0 as *const c_void) + } } #[derive(Debug, Clone, Copy)] pub enum SideExitReason { - UnknownNewarraySend(vm_opt_newarray_send_type), - UnknownCallType, - UnknownOpcode(u32), + UnhandledNewarraySend(vm_opt_newarray_send_type), + UnhandledDuparraySend(u64), + UnknownSpecialVariable(u64), + UnhandledHIRThrow, + UnhandledHIRInvokeBuiltin, + UnhandledHIRUnknown(InsnId), + UnhandledYARVInsn(u32), + UnhandledCallType(CallType), + UnhandledBlockArg, + TooManyKeywordParameters, FixnumAddOverflow, FixnumSubOverflow, FixnumMultOverflow, + FixnumLShiftOverflow, GuardType(Type), - GuardBitEquals(VALUE), + GuardShape(ShapeId), + ExpandArray, + GuardNotFrozen, + GuardNotShared, + GuardLess, + GuardGreaterEq, + GuardSuperMethodEntry, PatchPoint(Invariant), CalleeSideExit, ObjToStringFallback, - UnknownSpecialVariable(u64), - UnhandledDefinedType(usize), Interrupt, + BlockParamProxyNotIseqOrIfunc, + BlockParamProxyNotNil, + BlockParamProxyFallbackMiss, + BlockParamProxyProfileNotCovered, + BlockParamWbRequired, + StackOverflow, + FixnumModByZero, + FixnumDivByZero, + BoxFixnumOverflow, + SplatKwNotNilOrHash, + SplatKwPolymorphic, + SplatKwNotProfiled, + DirectiveInduced, + SendWhileTracing, + NoProfileSend, + InvokeBlockNotIfunc, +} + +/// Controls how a side exit triggers recompilation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Recompile { + /// Profile receiver + arguments from the stack (for sends without profile data). + ProfileSend { argc: i32 }, + /// Profile self from the CFP (for shape guard failures). + ProfileSelf, +} + +#[derive(Debug, Clone, Copy)] +pub enum MethodType { + Iseq, + Cfunc, + Attrset, + Ivar, + Bmethod, + Zsuper, + Alias, + Undefined, + NotImplemented, + Optimized, + Missing, + Refined, + Null, +} + +impl From<u32> for MethodType { + fn from(value: u32) -> Self { + match value { + VM_METHOD_TYPE_ISEQ => MethodType::Iseq, + VM_METHOD_TYPE_CFUNC => MethodType::Cfunc, + VM_METHOD_TYPE_ATTRSET => MethodType::Attrset, + VM_METHOD_TYPE_IVAR => MethodType::Ivar, + VM_METHOD_TYPE_BMETHOD => MethodType::Bmethod, + VM_METHOD_TYPE_ZSUPER => MethodType::Zsuper, + VM_METHOD_TYPE_ALIAS => MethodType::Alias, + VM_METHOD_TYPE_UNDEF => MethodType::Undefined, + VM_METHOD_TYPE_NOTIMPLEMENTED => MethodType::NotImplemented, + VM_METHOD_TYPE_OPTIMIZED => MethodType::Optimized, + VM_METHOD_TYPE_MISSING => MethodType::Missing, + VM_METHOD_TYPE_REFINED => MethodType::Refined, + _ => unreachable!("unknown send_without_block def_type: {}", value), + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum OptimizedMethodType { + Send, + Call, + BlockCall, + StructAref, + StructAset, +} + +impl From<u32> for OptimizedMethodType { + fn from(value: u32) -> Self { + match value { + OPTIMIZED_METHOD_TYPE_SEND => OptimizedMethodType::Send, + OPTIMIZED_METHOD_TYPE_CALL => OptimizedMethodType::Call, + OPTIMIZED_METHOD_TYPE_BLOCK_CALL => OptimizedMethodType::BlockCall, + OPTIMIZED_METHOD_TYPE_STRUCT_AREF => OptimizedMethodType::StructAref, + OPTIMIZED_METHOD_TYPE_STRUCT_ASET => OptimizedMethodType::StructAset, + _ => unreachable!("unknown send_without_block optimized method type: {}", value), + } + } } impl std::fmt::Display for SideExitReason { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - SideExitReason::UnknownOpcode(opcode) => write!(f, "UnknownOpcode({})", insn_name(*opcode as usize)), - SideExitReason::UnknownNewarraySend(VM_OPT_NEWARRAY_SEND_MAX) => write!(f, "UnknownNewarraySend(MAX)"), - SideExitReason::UnknownNewarraySend(VM_OPT_NEWARRAY_SEND_MIN) => write!(f, "UnknownNewarraySend(MIN)"), - SideExitReason::UnknownNewarraySend(VM_OPT_NEWARRAY_SEND_HASH) => write!(f, "UnknownNewarraySend(HASH)"), - SideExitReason::UnknownNewarraySend(VM_OPT_NEWARRAY_SEND_PACK) => write!(f, "UnknownNewarraySend(PACK)"), - SideExitReason::UnknownNewarraySend(VM_OPT_NEWARRAY_SEND_PACK_BUFFER) => write!(f, "UnknownNewarraySend(PACK_BUFFER)"), - SideExitReason::UnknownNewarraySend(VM_OPT_NEWARRAY_SEND_INCLUDE_P) => write!(f, "UnknownNewarraySend(INCLUDE_P)"), + SideExitReason::UnhandledYARVInsn(opcode) => write!(f, "UnhandledYARVInsn({})", insn_name(*opcode as usize)), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MAX) => write!(f, "UnhandledNewarraySend(MAX)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_MIN) => write!(f, "UnhandledNewarraySend(MIN)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_HASH) => write!(f, "UnhandledNewarraySend(HASH)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK) => write!(f, "UnhandledNewarraySend(PACK)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_PACK_BUFFER) => write!(f, "UnhandledNewarraySend(PACK_BUFFER)"), + SideExitReason::UnhandledNewarraySend(VM_OPT_NEWARRAY_SEND_INCLUDE_P) => write!(f, "UnhandledNewarraySend(INCLUDE_P)"), + SideExitReason::UnhandledDuparraySend(method_id) => write!(f, "UnhandledDuparraySend({method_id})"), SideExitReason::GuardType(guard_type) => write!(f, "GuardType({guard_type})"), - SideExitReason::GuardBitEquals(value) => write!(f, "GuardBitEquals({})", value.print(&PtrPrintMap::identity())), + SideExitReason::GuardNotShared => write!(f, "GuardNotShared"), SideExitReason::PatchPoint(invariant) => write!(f, "PatchPoint({invariant})"), _ => write!(f, "{self:?}"), } } } +/// Result of resolving the receiver type for method dispatch optimization. +/// Represents whether we know the receiver's class statically at compile-time, +/// have profiled type information, or know nothing about it. +pub enum ReceiverTypeResolution { + /// No profile information available for the receiver + NoProfile, + /// The receiver has a monomorphic profile (single type observed, guard needed) + Monomorphic { profiled_type: ProfiledType }, + /// The receiver is polymorphic (multiple types, none dominant) + Polymorphic, + /// The receiver has a skewed polymorphic profile (dominant type with some other types, guard needed) + SkewedPolymorphic { profiled_type: ProfiledType }, + /// More than N types seen with no clear winner + Megamorphic, + /// Megamorphic, but with a significant skew towards one type + SkewedMegamorphic { profiled_type: ProfiledType }, + /// The receiver's class is statically known at JIT compile-time (no guard needed) + StaticallyKnown { class: VALUE }, +} + +/// Reason why a send-ish instruction cannot be optimized from a fallback instruction +#[derive(Debug, Clone, Copy)] +pub enum SendFallbackReason { + SendWithoutBlockPolymorphic, + SendWithoutBlockMegamorphic, + SendWithoutBlockNoProfiles, + SendWithoutBlockCfuncNotVariadic, + SendWithoutBlockCfuncArrayVariadic, + SendWithoutBlockNotOptimizedMethodType(MethodType), + SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType), + SendWithoutBlockNotOptimizedNeedPermission, + SendWithoutBlockBopRedefined, + SendWithoutBlockOperandsNotFixnum, + SendWithoutBlockPolymorphicFallback, + SendDirectKeywordMismatch, + SendDirectKeywordCountMismatch, + SendDirectMissingKeyword, + SendDirectTooManyKeywords, + SendPolymorphic, + SendMegamorphic, + SendNoProfiles, + SendCfuncVariadic, + SendCfuncArrayVariadic, + SendNotOptimizedMethodType(MethodType), + SendNotOptimizedNeedPermission, + CCallWithFrameTooManyArgs, + ObjToStringNotString, + TooManyArgsForLir, + /// The Proc object for a BMETHOD is not defined by an ISEQ. (See `enum rb_block_type`.) + BmethodNonIseqProc, + /// Caller supplies too few or too many arguments than what the callee's parameters expects. + ArgcParamMismatch, + /// The call has at least one feature on the caller or callee side that the optimizer does not + /// support. + ComplexArgPass, + /// Caller has keyword arguments but callee doesn't expect them; need to convert to hash. + UnexpectedKeywordArgs, + /// A singleton class has been seen for the receiver class, so we skip the optimization + /// to avoid an invalidation loop. + SingletonClassSeen, + /// The super call is passed a block that the optimizer does not support. + SuperCallWithBlock, + /// When the `super` is in a block, finding the running CME for guarding requires a loop. Not + /// supported for now. + SuperFromBlock, + /// The profiled super class cannot be found. + SuperClassNotFound, + /// The `super` call uses a complex argument pattern that the optimizer does not support. + SuperComplexArgsPass, + /// The cached target of a `super` call could not be found. + SuperTargetNotFound, + /// Attempted to specialize a `super` call that doesn't have profile data. + SuperNoProfiles, + /// Cannot optimize the `super` call due to the target method. + SuperNotOptimizedMethodType(MethodType), + /// The `super` call is polymorpic. + SuperPolymorphic, + /// The `super` target call uses a complex argument pattern that the optimizer does not support. + SuperTargetComplexArgsPass, + /// The `invokeblock` instruction is not yet optimized in `type_specialize`. + InvokeBlockNotSpecialized, + /// The `sendforward` instruction (argument forwarding `...`) is not yet optimized in + /// `type_specialize`. + SendForwardNotSpecialized, + /// The `invokesuperforward` instruction (super with forwarding `...`) is not yet optimized in + /// `type_specialize`. + InvokeSuperForwardNotSpecialized, + /// The single-ractor-mode assumption could not be made. + SingleRactorModeRequired, + /// Initial fallback reason for every instruction, which should be mutated to + /// a more actionable reason when an attempt to specialize the instruction fails. + Uncategorized(ruby_vminsn_type), +} + +impl Display for SendFallbackReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + SendWithoutBlockPolymorphic => write!(f, "SendWithoutBlock: polymorphic call site"), + SendWithoutBlockMegamorphic => write!(f, "SendWithoutBlock: megamorphic call site"), + SendWithoutBlockNoProfiles => write!(f, "SendWithoutBlock: no profile data available"), + SendWithoutBlockCfuncNotVariadic => write!(f, "SendWithoutBlock: C function is not variadic"), + SendWithoutBlockCfuncArrayVariadic => write!(f, "SendWithoutBlock: C function expects array variadic"), + SendWithoutBlockNotOptimizedMethodType(method_type) => write!(f, "SendWithoutBlock: unsupported method type {:?}", method_type), + SendWithoutBlockNotOptimizedMethodTypeOptimized(opt_type) => write!(f, "SendWithoutBlock: unsupported optimized method type {:?}", opt_type), + SendWithoutBlockNotOptimizedNeedPermission => write!(f, "SendWithoutBlock: method private or protected and no FCALL"), + SendNotOptimizedNeedPermission => write!(f, "Send: method private or protected and no FCALL"), + SendWithoutBlockBopRedefined => write!(f, "SendWithoutBlock: basic operation was redefined"), + SendWithoutBlockOperandsNotFixnum => write!(f, "SendWithoutBlock: operands are not fixnums"), + SendWithoutBlockPolymorphicFallback => write!(f, "SendWithoutBlock: polymorphic fallback"), + SendDirectKeywordMismatch => write!(f, "SendDirect: keyword mismatch"), + SendDirectKeywordCountMismatch => write!(f, "SendDirect: keyword count mismatch"), + SendDirectMissingKeyword => write!(f, "SendDirect: missing keyword"), + SendDirectTooManyKeywords => write!(f, "SendDirect: too many keywords for fixnum bitmask"), + SendPolymorphic => write!(f, "Send: polymorphic call site"), + SendMegamorphic => write!(f, "Send: megamorphic call site"), + SendNoProfiles => write!(f, "Send: no profile data available"), + SendCfuncVariadic => write!(f, "Send: C function is variadic"), + SendCfuncArrayVariadic => write!(f, "Send: C function expects array variadic"), + SendNotOptimizedMethodType(method_type) => write!(f, "Send: unsupported method type {:?}", method_type), + CCallWithFrameTooManyArgs => write!(f, "CCallWithFrame: too many arguments"), + ObjToStringNotString => write!(f, "ObjToString: result is not a string"), + TooManyArgsForLir => write!(f, "Too many arguments for LIR"), + BmethodNonIseqProc => write!(f, "Bmethod: Proc object is not defined by an ISEQ"), + ArgcParamMismatch => write!(f, "Argument count does not match parameter count"), + ComplexArgPass => write!(f, "Complex argument passing"), + UnexpectedKeywordArgs => write!(f, "Unexpected Keyword Args"), + SingletonClassSeen => write!(f, "Singleton class previously created for receiver class"), + SuperFromBlock => write!(f, "super: call from within a block"), + SuperCallWithBlock => write!(f, "super: call made with a block"), + SuperClassNotFound => write!(f, "super: profiled class cannot be found"), + SuperComplexArgsPass => write!(f, "super: complex argument passing to `super` call"), + SuperNoProfiles => write!(f, "super: no profile data available"), + SuperNotOptimizedMethodType(method_type) => write!(f, "super: unsupported target method type {:?}", method_type), + SuperPolymorphic => write!(f, "super: polymorphic call site"), + SuperTargetNotFound => write!(f, "super: profiled target method cannot be found"), + SuperTargetComplexArgsPass => write!(f, "super: complex argument passing to `super` target call"), + InvokeBlockNotSpecialized => write!(f, "InvokeBlock: not yet specialized"), + SendForwardNotSpecialized => write!(f, "SendForward: not yet specialized"), + InvokeSuperForwardNotSpecialized => write!(f, "InvokeSuperForward: not yet specialized"), + SingleRactorModeRequired => write!(f, "Single-ractor mode required"), + Uncategorized(insn) => write!(f, "Uncategorized({})", insn_name(*insn as usize)), + } + } +} + +/// How a block is passed to a send-like instruction. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum BlockHandler { + /// Literal block ISEQ (e.g. `foo { ... }`) + BlockIseq(IseqPtr), + /// Block arg passed via &proc (e.g. `foo(&block)`) + BlockArg, +} + +/// Identifier used by LoadField/StoreField/LoadArg for HIR dumps. Variants +/// without an associated value name internal VM fields that we used to intern +/// as CRuby IDs just to print them; the `Id` variant carries a real CRuby ID +/// (e.g. local variable, ivar, struct field name). +#[allow(non_camel_case_types)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FieldName { + VM_ENV_DATA_INDEX_ME_CREF, + VM_ENV_DATA_INDEX_SPECVAL, + VM_ENV_DATA_INDEX_FLAGS, + RBASIC_FLAGS, + shape_id, + as_heap, + fields_obj, + thread_ptr, + len, + SelfParam, + Id(ID), +} + +impl std::fmt::Display for FieldName { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + use FieldName::*; + match self { + Id(id) if id_is_empty(*id) => f.write_str("<empty>"), + Id(id) => f.write_str(&id.contents_lossy()), + SelfParam => f.write_str("self"), + _ => write!(f, "{self:?}"), + } + } +} + +impl From<ID> for FieldName { + fn from(id: ID) -> Self { + FieldName::Id(id) + } +} + /// An instruction in the SSA IR. The output of an instruction is referred to by the index of /// the instruction ([`InsnId`]). SSA form enables this, and [`UnionFind`] ([`Function::find`]) /// helps with editing. #[derive(Debug, Clone)] pub enum Insn { + /// Comment that can be inserted into HIR for diagnostics. + Comment { message: String }, + Const { val: Const }, /// SSA block parameter. Also used for function parameters in the function's entry block. - Param { idx: usize }, + Param, + /// Load a function argument from the calling convention. + /// Used in JIT entry blocks. idx is the calling convention index, id is for display. + LoadArg { idx: u32, id: FieldName, val_type: Type }, + + /// Synthetic terminator for the entries superblock. Targets all entry blocks + /// so that CFG analyses see a single root. Not lowered to machine code. + Entries { targets: Vec<BlockId> }, StringCopy { val: InsnId, chilled: bool, state: InsnId }, StringIntern { val: InsnId, state: InsnId }, StringConcat { strings: Vec<InsnId>, state: InsnId }, + /// Call rb_str_getbyte with known-Fixnum index + StringGetbyte { string: InsnId, index: InsnId }, + StringSetbyteFixnum { string: InsnId, index: InsnId, value: InsnId }, + StringAppend { recv: InsnId, other: InsnId, state: InsnId }, + StringAppendCodepoint { recv: InsnId, other: InsnId, state: InsnId }, + StringEqual { left: InsnId, right: InsnId }, /// Combine count stack values into a regexp ToRegexp { opt: usize, values: Vec<InsnId>, state: InsnId }, /// Put special object (VMCORE, CBASE, etc.) based on value_type - PutSpecialObject { value_type: SpecialObjectType }, + PutSpecialObject { value_type: SpecialObjectType, state: InsnId }, /// Call `to_a` on `val` if the method is defined, or make a new array `[val]` otherwise. ToArray { val: InsnId, state: InsnId }, @@ -487,24 +886,75 @@ pub enum Insn { ToNewArray { val: InsnId, state: InsnId }, NewArray { elements: Vec<InsnId>, state: InsnId }, /// NewHash contains a vec of (key, value) pairs - NewHash { elements: Vec<(InsnId,InsnId)>, state: InsnId }, + NewHash { elements: Vec<InsnId>, state: InsnId }, NewRange { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, + NewRangeFixnum { low: InsnId, high: InsnId, flag: RangeType, state: InsnId }, ArrayDup { val: InsnId, state: InsnId }, + ArrayHash { elements: Vec<InsnId>, state: InsnId }, ArrayMax { elements: Vec<InsnId>, state: InsnId }, + ArrayMin { elements: Vec<InsnId>, state: InsnId }, + ArrayInclude { elements: Vec<InsnId>, target: InsnId, state: InsnId }, + ArrayPackBuffer { elements: Vec<InsnId>, fmt: InsnId, buffer: Option<InsnId>, state: InsnId }, + DupArrayInclude { ary: VALUE, target: InsnId, state: InsnId }, /// Extend `left` with the elements from `right`. `left` and `right` must both be `Array`. ArrayExtend { left: InsnId, right: InsnId, state: InsnId }, /// Push `val` onto `array`, where `array` is already `Array`. ArrayPush { array: InsnId, val: InsnId, state: InsnId }, - + ArrayAref { array: InsnId, index: InsnId }, + ArrayAset { array: InsnId, index: InsnId, val: InsnId }, + ArrayPop { array: InsnId, state: InsnId }, + /// Return the length of the array as a C `long` ([`types::CInt64`]) + ArrayLength { array: InsnId }, + /// Adjust potentially-negative index by the given length, returning the adjusted index. If + /// still negative, return a negative number, which indicates the index is still out-of-bounds. + AdjustBounds { index: InsnId, length: InsnId }, + + HashAref { hash: InsnId, key: InsnId, state: InsnId }, + HashAset { hash: InsnId, key: InsnId, val: InsnId, state: InsnId }, HashDup { val: InsnId, state: InsnId }, + /// Allocate an instance of the `val` object without calling `#initialize` on it. + /// This can: + /// * raise an exception if `val` is not a class + /// * run arbitrary code if `val` is a class with a custom allocator + ObjectAlloc { val: InsnId, state: InsnId }, + /// Allocate an instance of the `val` class without calling `#initialize` on it. + /// This requires that `class` has the default allocator (for example via `IsMethodCfunc`). + /// This won't raise or run arbitrary code because `class` has the default allocator. + ObjectAllocClass { class: VALUE, state: InsnId }, + /// Check if the value is truthy and "return" a C boolean. In reality, we will likely fuse this /// with IfTrue/IfFalse in the backend to generate jcc. Test { val: InsnId }, - /// Return C `true` if `val` is `Qnil`, else `false`. - IsNil { val: InsnId }, - Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId, state: InsnId }, + /// Return C `true` if `val`'s method on cd resolves to the cfunc. + IsMethodCfunc { val: InsnId, cd: *const rb_call_data, cfunc: *const u8, state: InsnId }, + /// Return C `true` if left == right + IsBitEqual { left: InsnId, right: InsnId }, + /// Return C `true` if left != right + IsBitNotEqual { left: InsnId, right: InsnId }, + /// Convert a C `bool` to a Ruby `Qtrue`/`Qfalse`. Same as `RBOOL` macro. + BoxBool { val: InsnId }, + /// Convert a C `long` to a Ruby `Fixnum`. Side exit on overflow. + BoxFixnum { val: InsnId, state: InsnId }, + UnboxFixnum { val: InsnId }, + FixnumAref { recv: InsnId, index: InsnId }, + // TODO(max): In iseq body types that are not ISEQ_TYPE_METHOD, rewrite to Constant false. + // `lep_level` is the lexical distance from this insn's iseq up to its local_iseq, used only + // for the DEFINED_YIELD op_type to materialize the local EP inline. Zero for other op_types. + Defined { op_type: usize, obj: VALUE, pushval: VALUE, v: InsnId, lep_level: u32, state: InsnId }, + GetConstant { klass: InsnId, id: ID, allow_nil: InsnId, state: InsnId }, GetConstantPath { ic: *const iseq_inline_constant_cache, state: InsnId }, + /// Kernel#block_given? but without pushing a frame. Similar to [`Insn::Defined`] with + /// `DEFINED_YIELD` + IsBlockGiven { lep: InsnId }, + /// Test the bit at index of val, a Fixnum. + /// Return Qtrue if the bit is set, else Qfalse. + FixnumBitCheck { val: InsnId, index: u8 }, + /// Return Qtrue if `val` is an instance of `class`, else Qfalse. + /// Equivalent to `class_search_ancestor(CLASS_OF(val), class)`. + IsA { val: InsnId, class: InsnId }, + /// `case`/`when`/`rescue` match check for `pattern` against `target`. + CheckMatch { target: InsnId, pattern: InsnId, flag: u32, state: InsnId }, /// Get a global variable named `id` GetGlobal { id: ID, state: InsnId }, @@ -512,20 +962,45 @@ pub enum Insn { SetGlobal { id: ID, val: InsnId, state: InsnId }, //NewObject? - /// Get an instance variable `id` from `self_val` - GetIvar { self_val: InsnId, id: ID, state: InsnId }, - /// Set `self_val`'s instance variable `id` to `val` - SetIvar { self_val: InsnId, id: ID, val: InsnId, state: InsnId }, + /// Get an instance variable `id` from `self_val`, using the inline cache `ic` if present + GetIvar { self_val: InsnId, id: ID, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, + /// Set `self_val`'s instance variable `id` to `val`, using the inline cache `ic` if present + SetIvar { self_val: InsnId, id: ID, val: InsnId, ic: *const iseq_inline_iv_cache_entry, state: InsnId }, /// Check whether an instance variable exists on `self_val` DefinedIvar { self_val: InsnId, id: ID, pushval: VALUE, state: InsnId }, - /// Get a local variable from a higher scope or the heap - GetLocal { level: u32, ep_offset: u32 }, + /// Load cfp->pc + LoadPC, + /// Load EC + LoadEC, + /// Load SP + LoadSP, + /// Load cfp->self + LoadSelf, + LoadField { recv: InsnId, id: FieldName, offset: i32, return_type: Type }, + /// Write `val` at an offset of `recv`. + /// When writing a Ruby object to a Ruby object, one must use GuardNotFrozen (or equivalent) before and WriteBarrier after. + StoreField { recv: InsnId, id: FieldName, offset: i32, val: InsnId }, + WriteBarrier { recv: InsnId, val: InsnId }, + + /// Check whether VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM is set in the (already loaded) environment flags. + /// Returns CBool (0/1). + IsBlockParamModified { flags: InsnId }, + /// Get the block parameter as a Proc. + GetBlockParam { level: u32, ep_offset: u32, state: InsnId }, /// Set a local variable in a higher scope or the heap SetLocal { level: u32, ep_offset: u32, val: InsnId }, GetSpecialSymbol { symbol_type: SpecialBackrefSymbol, state: InsnId }, GetSpecialNumber { nth: u64, state: InsnId }, + /// Get a class variable `id` + GetClassVar { id: ID, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + /// Set a class variable `id` to `val` + SetClassVar { id: ID, val: InsnId, ic: *const iseq_inline_cvar_cache_entry, state: InsnId }, + + /// Get the EP at the given level from the current CFP. + GetEP { level: u32 }, + /// Own a FrameState so that instructions can look up their dominating FrameState when /// generating deopt side-exits and frame reconstruction metadata. Does not directly generate /// any code. @@ -534,41 +1009,127 @@ pub enum Insn { /// Unconditional jump Jump(BranchEdge), - /// Conditional branch instructions - IfTrue { val: InsnId, target: BranchEdge }, - IfFalse { val: InsnId, target: BranchEdge }, + /// Conditional branch + CondBranch { val: InsnId, if_true: BranchEdge, if_false: BranchEdge }, + + /// Call a C function without pushing a frame + /// `name` and `owner` are for printing purposes only + CCall { cfunc: *const u8, recv: InsnId, args: Vec<InsnId>, name: ID, owner: VALUE, return_type: Type, elidable: bool }, + + /// Call a C function that pushes a frame + CCallWithFrame { + cd: *const rb_call_data, // cd for falling back to Send + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + block: Option<BlockHandler>, + }, - /// Call a C function - /// `name` is for printing purposes only - CCall { cfun: *const u8, args: Vec<InsnId>, name: ID, return_type: Type, elidable: bool }, + /// Call a variadic C function with signature: func(int argc, VALUE *argv, VALUE recv) + /// This handles frame setup, argv creation, and frame teardown all in one + CCallVariadic { + cfunc: *const u8, + recv: InsnId, + args: Vec<InsnId>, + cme: *const rb_callable_method_entry_t, + name: ID, + state: InsnId, + return_type: Type, + elidable: bool, + block: Option<BlockHandler>, + }, - /// Send without block with dynamic dispatch + /// Un-optimized fallback implementation (dynamic dispatch) for send-ish instructions /// Ignoring keyword arguments etc for now - SendWithoutBlock { self_val: InsnId, cd: *const rb_call_data, args: Vec<InsnId>, state: InsnId }, - Send { self_val: InsnId, cd: *const rb_call_data, blockiseq: IseqPtr, args: Vec<InsnId>, state: InsnId }, - SendWithoutBlockDirect { - self_val: InsnId, + Send { + recv: InsnId, + cd: *const rb_call_data, + block: Option<BlockHandler>, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + SendForward { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeSuper { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeSuperForward { + recv: InsnId, + cd: *const rb_call_data, + blockiseq: IseqPtr, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + InvokeBlock { + cd: *const rb_call_data, + args: Vec<InsnId>, + state: InsnId, + reason: SendFallbackReason, + }, + /// Optimized invokeblock for IFUNC block handlers. + /// Calls rb_vm_yield_with_cfunc directly instead of going through rb_vm_invokeblock. + InvokeBlockIfunc { + cd: *const rb_call_data, + block_handler: InsnId, + args: Vec<InsnId>, + state: InsnId, + }, + /// Call Proc#call optimized method type. + InvokeProc { + recv: InsnId, + args: Vec<InsnId>, + state: InsnId, + kw_splat: bool, + }, + + /// Optimized ISEQ call + SendDirect { + recv: InsnId, cd: *const rb_call_data, cme: *const rb_callable_method_entry_t, iseq: IseqPtr, args: Vec<InsnId>, + kw_bits: u32, + block: Option<BlockHandler>, state: InsnId, }, // Invoke a builtin function InvokeBuiltin { bf: rb_builtin_function, + recv: InsnId, args: Vec<InsnId>, state: InsnId, + leaf: bool, return_type: Option<Type>, // None for unannotated builtins }, + /// Set up frame. Remember the address as the JIT entry for the insn_idx in `jit_entry_insns()[jit_entry_idx]`. + EntryPoint { jit_entry_idx: Option<usize> }, /// Control flow instructions Return { val: InsnId }, /// Non-local control flow. See the throw YARV instruction - Throw { throw_state: u32, val: InsnId }, + Throw { throw_state: u32, val: InsnId, state: InsnId }, - /// Fixnum +, -, *, /, %, ==, !=, <, <=, >, >=, &, | + /// Fixnum +, -, *, /, %, ==, !=, <, <=, >, >=, &, |, ^, << FixnumAdd { left: InsnId, right: InsnId, state: InsnId }, FixnumSub { left: InsnId, right: InsnId, state: InsnId }, FixnumMult { left: InsnId, right: InsnId, state: InsnId }, @@ -582,41 +1143,395 @@ pub enum Insn { FixnumGe { left: InsnId, right: InsnId }, FixnumAnd { left: InsnId, right: InsnId }, FixnumOr { left: InsnId, right: InsnId }, - - // Distinct from `SendWithoutBlock` with `mid:to_s` because does not have a patch point for String to_s being redefined + FixnumXor { left: InsnId, right: InsnId }, + IntAnd { left: InsnId, right: InsnId }, + IntOr { left: InsnId, right: InsnId }, + FixnumLShift { left: InsnId, right: InsnId, state: InsnId }, + FixnumRShift { left: InsnId, right: InsnId }, + + /// Float arithmetic: delegates to rb_float_plus/minus/mul/div with GC preparation + FloatAdd { recv: InsnId, other: InsnId, state: InsnId }, + FloatSub { recv: InsnId, other: InsnId, state: InsnId }, + FloatMul { recv: InsnId, other: InsnId, state: InsnId }, + FloatDiv { recv: InsnId, other: InsnId, state: InsnId }, + /// Float#to_i: truncate float to integer via rb_jit_flo_to_i + FloatToInt { recv: InsnId, state: InsnId }, + + // Distinct from `Send` with `mid:to_s` because does not have a patch point for String to_s being redefined ObjToString { val: InsnId, cd: *const rb_call_data, state: InsnId }, AnyToString { val: InsnId, str: InsnId, state: InsnId }, + /// Refine the known type information of with additional type information. + /// Computes the intersection of the existing type and the new type. + RefineType { val: InsnId, new_type: Type }, + /// Return CBool[true] if val has type Type and CBool[false] otherwise. + HasType { val: InsnId, expected: Type }, + /// Side-exit if val doesn't have the expected type. - GuardType { val: InsnId, guard_type: Type, state: InsnId }, - /// Side-exit if val is not the expected VALUE. - GuardBitEquals { val: InsnId, expected: VALUE, state: InsnId }, + GuardType { val: InsnId, guard_type: Type, state: InsnId, recompile: Option<Recompile> }, + /// Side-exit if val is not the expected Const. + GuardBitEquals { val: InsnId, expected: Const, reason: SideExitReason, state: InsnId, recompile: Option<Recompile> }, + /// Side-exit if (val & mask) == 0 + GuardAnyBitSet { val: InsnId, mask: Const, mask_name: Option<ID>, reason: SideExitReason, state: InsnId }, + /// Side-exit if (val & mask) != 0 + GuardNoBitsSet { val: InsnId, mask: Const, mask_name: Option<ID>, reason: SideExitReason, state: InsnId }, + /// Side-exit if left is not greater than or equal to right (both operands are C long). + GuardGreaterEq { left: InsnId, right: InsnId, reason: SideExitReason, state: InsnId }, + /// Side-exit if left is not less than right (both operands are C long). + GuardLess { left: InsnId, right: InsnId, reason: SideExitReason, state: InsnId }, /// Generate no code (or padding if necessary) and insert a patch point /// that can be rewritten to a side exit when the Invariant is broken. PatchPoint { invariant: Invariant, state: InsnId }, /// Side-exit into the interpreter. - SideExit { state: InsnId, reason: SideExitReason }, + /// If recompile is not None, the side exit will profile and invalidate the ISEQ + /// so that it gets recompiled with the new profile data. + SideExit { state: InsnId, reason: SideExitReason, recompile: Option<Recompile> }, /// Increment a counter in ZJIT stats IncrCounter(Counter), + /// Increment a counter in ZJIT stats for the given counter pointer + IncrCounterPtr { counter_ptr: *mut u64 }, + /// Equivalent of RUBY_VM_CHECK_INTS. Automatically inserted by the compiler before jumps and /// return instructions. CheckInterrupts { state: InsnId }, + + BreakPoint, + + /// Only use this instruction in tests where you need to end a block with + /// a terminator, but don't ever expect the code to be executed. This + /// instruction should never be generated from iseq_to_hir + Unreachable, +} + +/// Macro that enumerates all operands of an Insn, dispatching to caller-provided +/// `$visit_one` macro for a single InsnId field and `$visit_many` macro for a +/// slice/Vec of InsnIds. Used by both `for_each_operand` and `for_each_operand_mut`. +macro_rules! for_each_operand_impl { + ($self:expr, $visit_one:ident, $visit_many:ident) => { + match $self { + Insn::Comment { .. } + | Insn::Const { .. } + | Insn::Param + | Insn::LoadArg { .. } + | Insn::Entries { .. } + | Insn::EntryPoint { .. } + | Insn::LoadPC + | Insn::LoadSP + | Insn::LoadEC + | Insn::GetEP { .. } + | Insn::LoadSelf + | Insn::BreakPoint | Insn::Unreachable + | Insn::IncrCounter(_) + | Insn::IncrCounterPtr { .. } => {} + + Insn::IsBlockGiven { lep } => { + $visit_one!(lep); + } + Insn::IsBlockParamModified { flags } => { + $visit_one!(flags); + } + Insn::CheckMatch { target, pattern, state, .. } => { + $visit_one!(target); + $visit_one!(pattern); + $visit_one!(state); + } + Insn::PatchPoint { state, .. } + | Insn::CheckInterrupts { state } + | Insn::PutSpecialObject { state, .. } + | Insn::GetBlockParam { state, .. } + | Insn::GetConstantPath { state, .. } => { + $visit_one!(state); + } + Insn::FixnumBitCheck { val, .. } => { + $visit_one!(val); + } + Insn::ArrayMax { elements, state, .. } + | Insn::ArrayMin { elements, state, .. } + | Insn::ArrayHash { elements, state, .. } + | Insn::NewHash { elements, state, .. } + | Insn::NewArray { elements, state, .. } => { + $visit_many!(elements); + $visit_one!(state); + } + Insn::ArrayInclude { elements, target, state, .. } => { + $visit_many!(elements); + $visit_one!(target); + $visit_one!(state); + } + Insn::ArrayPackBuffer { elements, fmt, buffer, state, .. } => { + $visit_many!(elements); + $visit_one!(fmt); + if let Some(buffer) = buffer { + $visit_one!(buffer); + } + $visit_one!(state); + } + Insn::DupArrayInclude { target, state, .. } => { + $visit_one!(target); + $visit_one!(state); + } + Insn::NewRange { low, high, state, .. } + | Insn::NewRangeFixnum { low, high, state, .. } => { + $visit_one!(low); + $visit_one!(high); + $visit_one!(state); + } + Insn::StringConcat { strings, state, .. } => { + $visit_many!(strings); + $visit_one!(state); + } + Insn::StringGetbyte { string, index } => { + $visit_one!(string); + $visit_one!(index); + } + Insn::StringSetbyteFixnum { string, index, value } => { + $visit_one!(string); + $visit_one!(index); + $visit_one!(value); + } + Insn::StringAppend { recv, other, state } + | Insn::StringAppendCodepoint { recv, other, state } => { + $visit_one!(recv); + $visit_one!(other); + $visit_one!(state); + } + Insn::StringEqual { left, right } => { + $visit_one!(left); + $visit_one!(right); + } + Insn::ToRegexp { values, state, .. } => { + $visit_many!(values); + $visit_one!(state); + } + Insn::RefineType { val, .. } + | Insn::HasType { val, .. } + | Insn::Return { val } + | Insn::Test { val } + | Insn::SetLocal { val, .. } + | Insn::BoxBool { val } => { + $visit_one!(val); + } + Insn::SetGlobal { val, state, .. } + | Insn::Defined { v: val, state, .. } + | Insn::StringIntern { val, state } + | Insn::StringCopy { val, state, .. } + | Insn::ObjectAlloc { val, state } + | Insn::GuardType { val, state, .. } + | Insn::GuardBitEquals { val, state, .. } + | Insn::GuardAnyBitSet { val, state, .. } + | Insn::GuardNoBitsSet { val, state, .. } + | Insn::ToArray { val, state } + | Insn::IsMethodCfunc { val, state, .. } + | Insn::ToNewArray { val, state } + | Insn::BoxFixnum { val, state } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::GuardGreaterEq { left, right, state, .. } + | Insn::GuardLess { left, right, state, .. } => { + $visit_one!(left); + $visit_one!(right); + $visit_one!(state); + } + Insn::Snapshot { state } => { + $visit_many!(state.stack); + $visit_many!(state.locals); + } + Insn::FixnumAdd { left, right, state } + | Insn::FixnumSub { left, right, state } + | Insn::FixnumMult { left, right, state } + | Insn::FixnumDiv { left, right, state } + | Insn::FixnumMod { left, right, state } + | Insn::ArrayExtend { left, right, state } + | Insn::FixnumLShift { left, right, state } => { + $visit_one!(left); + $visit_one!(right); + $visit_one!(state); + } + Insn::FloatAdd { recv, other, state } + | Insn::FloatSub { recv, other, state } + | Insn::FloatMul { recv, other, state } + | Insn::FloatDiv { recv, other, state } => { + $visit_one!(recv); + $visit_one!(other); + $visit_one!(state); + } + Insn::FloatToInt { recv, state } => { + $visit_one!(recv); + $visit_one!(state); + } + Insn::FixnumLt { left, right } + | Insn::FixnumLe { left, right } + | Insn::FixnumGt { left, right } + | Insn::FixnumGe { left, right } + | Insn::FixnumEq { left, right } + | Insn::FixnumNeq { left, right } + | Insn::FixnumAnd { left, right } + | Insn::FixnumOr { left, right } + | Insn::FixnumXor { left, right } + | Insn::IntAnd { left, right } + | Insn::IntOr { left, right } + | Insn::FixnumRShift { left, right } + | Insn::IsBitEqual { left, right } + | Insn::IsBitNotEqual { left, right } => { + $visit_one!(left); + $visit_one!(right); + } + Insn::Jump(BranchEdge { args, .. }) => { + $visit_many!(args); + } + Insn::CondBranch { val, if_true: BranchEdge { args: true_args, .. }, if_false: BranchEdge { args: false_args, .. } } => { + $visit_one!(val); + $visit_many!(true_args); + $visit_many!(false_args); + } + Insn::ArrayDup { val, state } + | Insn::Throw { val, state, .. } + | Insn::HashDup { val, state } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::ArrayAref { array, index } => { + $visit_one!(array); + $visit_one!(index); + } + Insn::ArrayAset { array, index, val } => { + $visit_one!(array); + $visit_one!(index); + $visit_one!(val); + } + Insn::ArrayPop { array, state } => { + $visit_one!(array); + $visit_one!(state); + } + Insn::ArrayLength { array } => { + $visit_one!(array); + } + Insn::AdjustBounds { index, length } => { + $visit_one!(index); + $visit_one!(length); + } + Insn::HashAref { hash, key, state } => { + $visit_one!(hash); + $visit_one!(key); + $visit_one!(state); + } + Insn::HashAset { hash, key, val, state } => { + $visit_one!(hash); + $visit_one!(key); + $visit_one!(val); + $visit_one!(state); + } + Insn::Send { recv, args, state, .. } + | Insn::SendForward { recv, args, state, .. } + | Insn::CCallVariadic { recv, args, state, .. } + | Insn::CCallWithFrame { recv, args, state, .. } + | Insn::SendDirect { recv, args, state, .. } + | Insn::InvokeBuiltin { recv, args, state, .. } + | Insn::InvokeSuper { recv, args, state, .. } + | Insn::InvokeSuperForward { recv, args, state, .. } + | Insn::InvokeProc { recv, args, state, .. } => { + $visit_one!(recv); + $visit_many!(args); + $visit_one!(state); + } + Insn::InvokeBlock { args, state, .. } => { + $visit_many!(args); + $visit_one!(state); + } + Insn::InvokeBlockIfunc { block_handler, args, state, .. } => { + $visit_one!(block_handler); + $visit_many!(args); + $visit_one!(state); + } + Insn::CCall { recv, args, .. } => { + $visit_one!(recv); + $visit_many!(args); + } + Insn::GetIvar { self_val, state, .. } + | Insn::DefinedIvar { self_val, state, .. } => { + $visit_one!(self_val); + $visit_one!(state); + } + Insn::GetConstant { klass, allow_nil, state, .. } => { + $visit_one!(klass); + $visit_one!(allow_nil); + $visit_one!(state); + } + Insn::SetIvar { self_val, val, state, .. } => { + $visit_one!(self_val); + $visit_one!(val); + $visit_one!(state); + } + Insn::GetClassVar { state, .. } => { + $visit_one!(state); + } + Insn::SetClassVar { val, state, .. } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::ArrayPush { array, val, state } => { + $visit_one!(array); + $visit_one!(val); + $visit_one!(state); + } + Insn::ObjToString { val, state, .. } => { + $visit_one!(val); + $visit_one!(state); + } + Insn::AnyToString { val, str, state, .. } => { + $visit_one!(val); + $visit_one!(str); + $visit_one!(state); + } + Insn::LoadField { recv, .. } => { + $visit_one!(recv); + } + Insn::StoreField { recv, val, .. } + | Insn::WriteBarrier { recv, val } => { + $visit_one!(recv); + $visit_one!(val); + } + Insn::GetGlobal { state, .. } + | Insn::GetSpecialSymbol { state, .. } + | Insn::GetSpecialNumber { state, .. } + | Insn::ObjectAllocClass { state, .. } + | Insn::SideExit { state, .. } => { + $visit_one!(state); + } + Insn::UnboxFixnum { val } => { + $visit_one!(val); + } + Insn::FixnumAref { recv, index } => { + $visit_one!(recv); + $visit_one!(index); + } + Insn::IsA { val, class } => { + $visit_one!(val); + $visit_one!(class); + } + } + }; } impl Insn { /// Not every instruction returns a value. Return true if the instruction does and false otherwise. pub fn has_output(&self) -> bool { match self { - Insn::Jump(_) - | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::Return { .. } - | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::ArrayExtend { .. } + Insn::Comment { .. } + | Insn::Jump(_) + | Insn::Entries { .. } + | Insn::CondBranch { .. } | Insn::EntryPoint { .. } | Insn::Return { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetGlobal { .. } - | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_) - | Insn::CheckInterrupts { .. } => false, + | Insn::SetLocal { .. } | Insn::Throw { .. } | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::BreakPoint | Insn::Unreachable + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } + | Insn::ArrayAset { .. } => false, _ => true, } } @@ -624,59 +1539,295 @@ impl Insn { /// Return true if the instruction ends a basic block and false otherwise. pub fn is_terminator(&self) -> bool { match self { - Insn::Jump(_) | Insn::Return { .. } | Insn::SideExit { .. } | Insn::Throw { .. } => true, + Insn::Unreachable | Insn::CondBranch { .. } | Insn::Jump(_) | Insn::Entries { .. } | Insn::Return { .. } | Insn::SideExit { .. } | Insn::Throw { .. } => true, + _ => false, + } + } + + /// Return true if the instruction is a jump (has successor blocks in the CFG). + pub fn is_jump(&self) -> bool { + match self { + Insn::CondBranch { .. } | Insn::Jump(_) | Insn::Entries { .. } => true, _ => false, } } - pub fn print<'a>(&self, ptr_map: &'a PtrPrintMap) -> InsnPrinter<'a> { - InsnPrinter { inner: self.clone(), ptr_map } + /// Call `f` on each operand (InsnId) of this instruction. + pub fn for_each_operand(&self, mut f: impl FnMut(InsnId)) { + macro_rules! visit_one { ($id:expr) => { f(*$id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id) } }; } + for_each_operand_impl!(self, visit_one, visit_many); } - /// Return true if the instruction needs to be kept around. For example, if the instruction - /// might have a side effect, or if the instruction may raise an exception. - fn has_effects(&self) -> bool { - match self { - Insn::Const { .. } => false, - Insn::Param { .. } => false, - Insn::StringCopy { .. } => false, - Insn::NewArray { .. } => false, - // NewHash's operands may be hashed and compared for equality, which could have - // side-effects. - Insn::NewHash { elements, .. } => elements.len() > 0, - Insn::ArrayDup { .. } => false, - Insn::HashDup { .. } => false, - Insn::Test { .. } => false, - Insn::Snapshot { .. } => false, - Insn::FixnumAdd { .. } => false, - Insn::FixnumSub { .. } => false, - Insn::FixnumMult { .. } => false, - // TODO(max): Consider adding a Guard that the rhs is non-zero before Div and Mod - // Div *is* critical unless we can prove the right hand side != 0 - // Mod *is* critical unless we can prove the right hand side != 0 - Insn::FixnumEq { .. } => false, - Insn::FixnumNeq { .. } => false, - Insn::FixnumLt { .. } => false, - Insn::FixnumLe { .. } => false, - Insn::FixnumGt { .. } => false, - Insn::FixnumGe { .. } => false, - Insn::FixnumAnd { .. } => false, - Insn::FixnumOr { .. } => false, - Insn::GetLocal { .. } => false, - Insn::IsNil { .. } => false, - Insn::CCall { elidable, .. } => !elidable, - // TODO: NewRange is effects free if we can prove the two ends to be Fixnum, - // but we don't have type information here in `impl Insn`. See rb_range_new(). - Insn::NewRange { .. } => true, - _ => true, + /// Call `f` on a mutable reference to each operand (InsnId) of this instruction. + pub fn for_each_operand_mut(&mut self, mut f: impl FnMut(&mut InsnId)) { + macro_rules! visit_one { ($id:expr) => { f($id) }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter_mut() { f(id) } }; } + for_each_operand_impl!(self, visit_one, visit_many); + } + + /// Call `f` on each operand, short-circuiting on the first error. + pub fn try_for_each_operand<E>(&self, mut f: impl FnMut(InsnId) -> Result<(), E>) -> Result<(), E> { + macro_rules! visit_one { ($id:expr) => { f(*$id)? }; } + macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id)? } }; } + for_each_operand_impl!(self, visit_one, visit_many); + Ok(()) + } + + pub fn print<'a>(&self, ptr_map: &'a PtrPrintMap, iseq: Option<IseqPtr>) -> InsnPrinter<'a> { + InsnPrinter { inner: self.clone(), ptr_map, iseq } + } + + // TODO(Jacob): Model SP. ie, all allocations modify stack size but using the effect for stack modification feels excessive + // TODO(Jacob): Add sideeffect failure bit + fn effects_of(&self) -> Effect { + const allocates: Effect = Effect::read_write(abstract_heaps::PC.union(abstract_heaps::Allocator), abstract_heaps::Allocator); + match &self { + Insn::Comment { .. } => effects::Empty, + Insn::Const { .. } => effects::Empty, + Insn::Param { .. } => effects::Empty, + Insn::LoadArg { .. } => effects::Empty, + Insn::StringCopy { .. } => allocates, + Insn::StringIntern { .. } => effects::Any, + Insn::StringConcat { .. } => effects::Any, + Insn::StringGetbyte { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::StringSetbyteFixnum { .. } => effects::Any, + Insn::StringAppend { .. } => effects::Any, + Insn::StringAppendCodepoint { .. } => effects::Any, + Insn::StringEqual { .. } => Effect::write(abstract_heaps::Allocator), + Insn::ToRegexp { .. } => effects::Any, + Insn::PutSpecialObject { .. } => effects::Any, + Insn::ToArray { .. } => effects::Any, + Insn::ToNewArray { .. } => effects::Any, + Insn::NewArray { .. } => allocates, + Insn::NewHash { elements, .. } => { + // NewHash's operands may be hashed and compared for equality, which could have + // side-effects. Empty hashes are definitely elidable. + if elements.is_empty() { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::NewRange { .. } => effects::Any, + Insn::NewRangeFixnum { .. } => allocates, + Insn::ArrayDup { .. } => allocates, + Insn::ArrayHash { .. } => effects::Any, + Insn::ArrayMax { .. } => effects::Any, + Insn::ArrayMin { .. } => effects::Any, + Insn::ArrayInclude { .. } => effects::Any, + Insn::ArrayPackBuffer { .. } => effects::Any, + Insn::DupArrayInclude { .. } => effects::Any, + Insn::ArrayExtend { .. } => effects::Any, + Insn::ArrayPush { .. } => effects::Any, + Insn::ArrayAref { .. } => effects::Any, + Insn::ArrayAset { .. } => effects::Any, + Insn::ArrayPop { .. } => effects::Any, + Insn::ArrayLength { .. } => Effect::write(abstract_heaps::Empty), + Insn::AdjustBounds { .. } => effects::Empty, + Insn::HashAref { .. } => effects::Any, + Insn::HashAset { .. } => effects::Any, + Insn::HashDup { .. } => allocates, + Insn::ObjectAlloc { .. } => effects::Any, + Insn::ObjectAllocClass { .. } => allocates, + Insn::Test { .. } => effects::Empty, + Insn::IsMethodCfunc { .. } => effects::Any, + Insn::IsBitEqual { .. } => effects::Empty, + Insn::IsBitNotEqual { .. } => effects::Empty, + Insn::BoxBool { .. } => effects::Empty, + Insn::BoxFixnum { .. } => effects::Empty, + Insn::UnboxFixnum { .. } => effects::Any, + Insn::FixnumAref { .. } => effects::Empty, + Insn::Defined { .. } => effects::Any, + Insn::GetConstant { .. } => effects::Any, + Insn::GetConstantPath { .. } => effects::Any, + Insn::IsBlockGiven { .. } => Effect::read_write(abstract_heaps::Other, abstract_heaps::Empty), + Insn::FixnumBitCheck { .. } => effects::Empty, + // IsA needs to read the class of the value and traverse the class hierarchy, which we model as reading from Memory. + Insn::IsA { .. } => Effect::read_write(abstract_heaps::Memory, abstract_heaps::Empty), + Insn::GetGlobal { .. } => effects::Any, + Insn::SetGlobal { .. } => effects::Any, + Insn::GetIvar { .. } => effects::Any, + Insn::SetIvar { .. } => effects::Any, + Insn::DefinedIvar { .. } => effects::Any, + Insn::LoadPC { .. } => Effect::read_write(abstract_heaps::PC, abstract_heaps::Empty), + Insn::LoadEC { .. } => effects::Empty, + Insn::LoadSP { .. } => effects::Empty, + // GetEP reads from the current frame pointer (abstract_heaps::Frame) and also traverses previous frames too. + Insn::GetEP { .. } => Effect::read_write(abstract_heaps::Memory, abstract_heaps::Empty), + Insn::LoadSelf { .. } => Effect::read_write(abstract_heaps::Frame, abstract_heaps::Empty), + Insn::LoadField { .. } => Effect::read_write(abstract_heaps::Memory, abstract_heaps::Empty), + Insn::StoreField { .. } => effects::Any, + // TODO: Refine CheckMatch effects by flag. + Insn::CheckMatch { .. } => effects::Any, + // WriteBarrier can write to object flags and mark bits in Allocator memory. + // This is why WriteBarrier writes to the "Memory" effect. We do not yet have a more granular specialization for flags + Insn::WriteBarrier { .. } => Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Allocator.union(abstract_heaps::Memory)), + Insn::SetLocal { .. } => effects::Any, + Insn::GetSpecialSymbol { .. } => effects::Any, + Insn::GetSpecialNumber { .. } => effects::Any, + Insn::GetClassVar { .. } => effects::Any, + Insn::SetClassVar { .. } => effects::Any, + Insn::IsBlockParamModified { .. } => effects::Empty, + Insn::GetBlockParam { .. } => effects::Any, + Insn::Snapshot { .. } => effects::Empty, + Insn::Jump(_) => effects::Any, + Insn::CondBranch { .. } => effects::Any, + Insn::CCall { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallWithFrame { elidable, .. } => { + if *elidable { + Effect::write(abstract_heaps::Allocator) + } + else { + effects::Any + } + }, + Insn::CCallVariadic { .. } => effects::Any, + Insn::Send { .. } => effects::Any, + Insn::SendForward { .. } => effects::Any, + Insn::InvokeSuper { .. } => effects::Any, + Insn::InvokeSuperForward { .. } => effects::Any, + Insn::InvokeBlock { .. } => effects::Any, + Insn::InvokeBlockIfunc { .. } => effects::Any, + Insn::SendDirect { .. } => effects::Any, + Insn::InvokeBuiltin { .. } => effects::Any, + Insn::EntryPoint { .. } => effects::Any, + Insn::Return { .. } => effects::Any, + Insn::Throw { .. } => effects::Any, + Insn::FixnumAdd { .. } => effects::Empty, + Insn::FixnumSub { .. } => effects::Empty, + Insn::FixnumMult { .. } => effects::Empty, + Insn::FixnumDiv { .. } => effects::Any, + Insn::FixnumMod { .. } => effects::Any, + Insn::FloatAdd { .. } => effects::Any, + Insn::FloatSub { .. } => effects::Any, + Insn::FloatMul { .. } => effects::Any, + Insn::FloatDiv { .. } => effects::Any, + Insn::FloatToInt { .. } => effects::Any, + Insn::FixnumEq { .. } => effects::Empty, + Insn::FixnumNeq { .. } => effects::Empty, + Insn::FixnumLt { .. } => effects::Empty, + Insn::FixnumLe { .. } => effects::Empty, + Insn::FixnumGt { .. } => effects::Empty, + Insn::FixnumGe { .. } => effects::Empty, + Insn::FixnumAnd { .. } => effects::Empty, + Insn::FixnumOr { .. } => effects::Empty, + Insn::FixnumXor { .. } => effects::Empty, + Insn::IntAnd { .. } => effects::Empty, + Insn::IntOr { .. } => effects::Empty, + Insn::FixnumLShift { .. } => effects::Empty, + Insn::FixnumRShift { .. } => effects::Empty, + Insn::ObjToString { .. } => effects::Any, + Insn::AnyToString { .. } => effects::Any, + Insn::GuardType { guard_type, .. } + => Effect::read_write( + if guard_type.is_subtype(types::Immediate) { abstract_heaps::Empty } else { abstract_heaps::Memory }, + abstract_heaps::Control + ), + Insn::GuardBitEquals { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardAnyBitSet { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardNoBitsSet { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardGreaterEq { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::GuardLess { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), + Insn::PatchPoint { .. } => Effect::read_write(abstract_heaps::PatchPoint, abstract_heaps::Control), + Insn::SideExit { .. } => effects::Any, + Insn::IncrCounter(_) => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Other), + Insn::IncrCounterPtr { .. } => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Other), + Insn::CheckInterrupts { .. } => Effect::read_write(abstract_heaps::InterruptFlag, abstract_heaps::Control), + Insn::InvokeProc { .. } => effects::Any, + Insn::RefineType { .. } => effects::Empty, + Insn::HasType { expected, .. } + => Effect::read_write( + if expected.is_subtype(types::Immediate) { abstract_heaps::Empty } else { abstract_heaps::Memory }, + abstract_heaps::Empty + ), + Insn::Entries { .. } => effects::Any, + Insn::BreakPoint | Insn::Unreachable => Effect::read_write(abstract_heaps::Empty, abstract_heaps::Control), } } + + /// Return true if we can safely omit the instruction. This occurs when one of the following + /// conditions are met. + /// 1. The instruction does not write anything. + /// 2. The instruction only allocates and writes nothing else. + /// Calling the effects of our instruction `insn_effects`, we need: + /// `effects::Empty` to include `insn_effects.write` or `effects::Allocator` to include + /// `insn_effects.write`. + /// We can simplify this to `effects::Empty.union(effects::Allocator).includes(insn_effects.write)`. + /// But the union of `Allocator` and `Empty` is simply `Allocator`, so our entire function + /// collapses to `effects::Allocator.includes(insn_effects.write)`. + /// Note: These are restrictions on the `write` `EffectSet` only. Even instructions with + /// `read: effects::Any` could potentially be omitted. + fn is_elidable(&self) -> bool { + // Comments intentionally have no semantic effect, but they are diagnostics that should + // survive DCE so optimized HIR dumps retain the information callers inserted. + if matches!(self, Insn::Comment { .. }) { + return false; + } + + abstract_heaps::Allocator.includes(self.effects_of().write_bits()) + } } /// Print adaptor for [`Insn`]. See [`PtrPrintMap`]. pub struct InsnPrinter<'a> { inner: Insn, ptr_map: &'a PtrPrintMap, + iseq: Option<IseqPtr>, +} + +fn get_local_var_id(iseq: IseqPtr, level: u32, ep_offset: u32) -> ID { + let mut current_iseq = iseq; + for _ in 0..level { + current_iseq = unsafe { rb_get_iseq_body_parent_iseq(current_iseq) }; + } + let local_idx = ep_offset_to_local_idx(current_iseq, ep_offset); + unsafe { rb_zjit_local_id(current_iseq, local_idx.try_into().unwrap()) } +} + +/// Get the name of a local variable given iseq, level, and ep_offset. +/// Returns +/// - `":name"` if iseq is available and name is a real identifier, +/// - `"<empty>"` for anonymous locals. +/// - `None` if iseq is not available. +/// (When `Insn` is printed in a panic/debug message the `Display::fmt` method is called, which can't access an iseq.) +/// +/// This mimics local_var_name() from iseq.c. +fn get_local_var_name_for_printer(iseq: Option<IseqPtr>, level: u32, ep_offset: u32) -> Option<String> { + let id = get_local_var_id(iseq?, level, ep_offset); + + if id_is_empty(id) { + return Some(String::from("<empty>")); + } + + Some(format!(":{}", id.contents_lossy())) +} + + +fn id_is_empty(id: ID) -> bool { + id.0 == 0 || unsafe { rb_id2str(id) } == Qfalse +} + +/// Construct a qualified method name for display/debug output. +/// Returns strings like "Array#length" for instance methods or "Foo.bar" for singleton methods. +fn qualified_method_name(class: VALUE, method_id: ID) -> String { + let method_name = method_id.contents_lossy(); + // rb_zjit_singleton_class_p also checks if it's a class + if unsafe { rb_zjit_singleton_class_p(class) } { + let class_name = get_class_name(unsafe { rb_class_attached_object(class) }); + format!("{class_name}.{method_name}") + } else { + let class_name = get_class_name(class); + format!("{class_name}#{method_name}") + } } static REGEXP_FLAGS: &[(u32, &str)] = &[ @@ -690,8 +1841,19 @@ static REGEXP_FLAGS: &[(u32, &str)] = &[ impl<'a> std::fmt::Display for InsnPrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match &self.inner { + Insn::Comment { message } => write!(f, "# {message}"), Insn::Const { val } => { write!(f, "Const {}", val.print(self.ptr_map)) } - Insn::Param { idx } => { write!(f, "Param {idx}") } + Insn::Param => { write!(f, "Param") } + Insn::LoadArg { idx, id, .. } => { write!(f, "LoadArg :{id}@{idx}") } + Insn::Entries { targets } => { + write!(f, "Entries")?; + let mut prefix = " "; + for target in targets { + write!(f, "{prefix}{target}")?; + prefix = ", "; + } + Ok(()) + } Insn::NewArray { elements, .. } => { write!(f, "NewArray")?; let mut prefix = " "; @@ -701,18 +1863,38 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { } Ok(()) } + Insn::ArrayAref { array, index, .. } => { + write!(f, "ArrayAref {array}, {index}") + } + Insn::ArrayAset { array, index, val, ..} => { + write!(f, "ArrayAset {array}, {index}, {val}") + } + Insn::ArrayPop { array, .. } => { + write!(f, "ArrayPop {array}") + } + Insn::ArrayLength { array } => { + write!(f, "ArrayLength {array}") + } + Insn::AdjustBounds { index, length } => { + write!(f, "AdjustBounds {index}, {length}") + } Insn::NewHash { elements, .. } => { write!(f, "NewHash")?; let mut prefix = " "; - for (key, value) in elements { - write!(f, "{prefix}{key}: {value}")?; - prefix = ", "; + for chunk in elements.chunks(2) { + if let [key, value] = chunk { + write!(f, "{prefix}{key}: {value}")?; + prefix = ", "; + } } Ok(()) } Insn::NewRange { low, high, flag, .. } => { write!(f, "NewRange {low} {flag} {high}") } + Insn::NewRangeFixnum { low, high, flag, .. } => { + write!(f, "NewRangeFixnum {low} {flag} {high}") + } Insn::ArrayMax { elements, .. } => { write!(f, "ArrayMax")?; let mut prefix = " "; @@ -722,8 +1904,56 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { } Ok(()) } + Insn::ArrayMin { elements, .. } => { + write!(f, "ArrayMin")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayHash { elements, .. } => { + write!(f, "ArrayHash")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + Ok(()) + } + Insn::ArrayInclude { elements, target, .. } => { + write!(f, "ArrayInclude")?; + let mut prefix = " "; + for element in elements { + write!(f, "{prefix}{element}")?; + prefix = ", "; + } + write!(f, " | {target}") + } + Insn::ArrayPackBuffer { elements, fmt, buffer, .. } => { + write!(f, "ArrayPackBuffer ")?; + for element in elements { + write!(f, "{element}, ")?; + } + write!(f, "fmt: {fmt}")?; + if let Some(buffer) = buffer { + write!(f, ", buf: {buffer}")?; + } + Ok(()) + } + Insn::DupArrayInclude { ary, target, .. } => { + write!(f, "DupArrayInclude {} | {}", ary.print(self.ptr_map), target) + } Insn::ArrayDup { val, .. } => { write!(f, "ArrayDup {val}") } Insn::HashDup { val, .. } => { write!(f, "HashDup {val}") } + Insn::HashAref { hash, key, .. } => { write!(f, "HashAref {hash}, {key}")} + Insn::HashAset { hash, key, val, .. } => { write!(f, "HashAset {hash}, {key}, {val}")} + Insn::ObjectAlloc { val, .. } => { write!(f, "ObjectAlloc {val}") } + &Insn::ObjectAllocClass { class, .. } => { + let class_name = get_class_name(class); + write!(f, "ObjectAllocClass {class_name}:{}", class.print(self.ptr_map)) + } Insn::StringCopy { val, .. } => { write!(f, "StringCopy {val}") } Insn::StringConcat { strings, .. } => { write!(f, "StringConcat")?; @@ -735,6 +1965,21 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Ok(()) } + Insn::StringGetbyte { string, index, .. } => { + write!(f, "StringGetbyte {string}, {index}") + } + Insn::StringSetbyteFixnum { string, index, value, .. } => { + write!(f, "StringSetbyteFixnum {string}, {index}, {value}") + } + Insn::StringAppend { recv, other, .. } => { + write!(f, "StringAppend {recv}, {other}") + } + Insn::StringAppendCodepoint { recv, other, .. } => { + write!(f, "StringAppendCodepoint {recv}, {other}") + } + Insn::StringEqual { left, right } => { + write!(f, "StringEqual {left}, {right}") + } Insn::ToRegexp { values, opt, .. } => { write!(f, "ToRegexp")?; let mut prefix = " "; @@ -758,47 +2003,115 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Ok(()) } Insn::Test { val } => { write!(f, "Test {val}") } - Insn::IsNil { val } => { write!(f, "IsNil {val}") } + Insn::IsMethodCfunc { val, cd, .. } => { write!(f, "IsMethodCFunc {val}, :{}", ruby_call_method_name(*cd)) } + Insn::IsBitEqual { left, right } => write!(f, "IsBitEqual {left}, {right}"), + Insn::IsBitNotEqual { left, right } => write!(f, "IsBitNotEqual {left}, {right}"), + Insn::BoxBool { val } => write!(f, "BoxBool {val}"), + Insn::BoxFixnum { val, .. } => write!(f, "BoxFixnum {val}"), + Insn::UnboxFixnum { val } => write!(f, "UnboxFixnum {val}"), + Insn::FixnumAref { recv, index } => write!(f, "FixnumAref {recv}, {index}"), Insn::Jump(target) => { write!(f, "Jump {target}") } - Insn::IfTrue { val, target } => { write!(f, "IfTrue {val}, {target}") } - Insn::IfFalse { val, target } => { write!(f, "IfFalse {val}, {target}") } - Insn::SendWithoutBlock { self_val, cd, args, .. } => { - write!(f, "SendWithoutBlock {self_val}, :{}", ruby_call_method_name(*cd))?; + Insn::CondBranch { val, if_true, if_false } => { write!(f, "CondBranch {val}, {if_true}, {if_false}") }, + Insn::SendDirect { recv, cme, iseq, args, block, .. } => { + let blockiseq = block.map(|bh| match bh { BlockHandler::BlockIseq(iseq) => iseq, BlockHandler::BlockArg => unreachable!() }); + let method_name = unsafe { (**cme).called_id }; + write!(f, "SendDirect {recv}, {:p}, :{} ({:?})", self.ptr_map.map_ptr(&blockiseq), method_name, self.ptr_map.map_ptr(iseq))?; for arg in args { write!(f, ", {arg}")?; } Ok(()) } - Insn::SendWithoutBlockDirect { self_val, cd, iseq, args, .. } => { - write!(f, "SendWithoutBlockDirect {self_val}, :{} ({:?})", ruby_call_method_name(*cd), self.ptr_map.map_ptr(iseq))?; + Insn::Send { recv, cd, args, block, reason, .. } => { + // For tests, we want to check HIR snippets textually. Addresses change + // between runs, making tests fail. Instead, pick an arbitrary hex value to + // use as a "pointer" so we can check the rest of the HIR. + match *block { + Some(BlockHandler::BlockIseq(blockiseq)) => + write!(f, "Send {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?, + Some(BlockHandler::BlockArg) => + write!(f, "Send {recv}, &block, :{}", ruby_call_method_name(*cd))?, + None => + write!(f, "Send {recv}, :{}", ruby_call_method_name(*cd))?, + } for arg in args { write!(f, ", {arg}")?; } + write!(f, " # SendFallbackReason: {reason}")?; Ok(()) } - Insn::Send { self_val, cd, args, blockiseq, .. } => { - // For tests, we want to check HIR snippets textually. Addresses change - // between runs, making tests fail. Instead, pick an arbitrary hex value to - // use as a "pointer" so we can check the rest of the HIR. - write!(f, "Send {self_val}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?; + Insn::SendForward { recv, cd, args, blockiseq, reason, .. } => { + write!(f, "SendForward {recv}, {:p}, :{}", self.ptr_map.map_ptr(blockiseq), ruby_call_method_name(*cd))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeSuper { recv, blockiseq, args, reason, .. } => { + write!(f, "InvokeSuper {recv}, {:p}", self.ptr_map.map_ptr(blockiseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeSuperForward { recv, blockiseq, args, reason, .. } => { + write!(f, "InvokeSuperForward {recv}, {:p}", self.ptr_map.map_ptr(blockiseq))?; + for arg in args { + write!(f, ", {arg}")?; + } + write!(f, " # SendFallbackReason: {reason}")?; + Ok(()) + } + Insn::InvokeBlock { args, reason, .. } => { + write!(f, "InvokeBlock")?; for arg in args { write!(f, ", {arg}")?; } + write!(f, " # SendFallbackReason: {reason}")?; Ok(()) } - Insn::InvokeBuiltin { bf, args, .. } => { - write!(f, "InvokeBuiltin {}", unsafe { CStr::from_ptr(bf.name) }.to_str().unwrap())?; + Insn::InvokeBlockIfunc { block_handler, args, .. } => { + write!(f, "InvokeBlockIfunc {block_handler}")?; for arg in args { write!(f, ", {arg}")?; } Ok(()) } + Insn::InvokeProc { recv, args, kw_splat, .. } => { + write!(f, "InvokeProc {recv}")?; + for arg in args { + write!(f, ", {arg}")?; + } + if *kw_splat { + write!(f, ", kw_splat")?; + } + Ok(()) + } + Insn::InvokeBuiltin { bf, args, leaf, .. } => { + let bf_name = unsafe { CStr::from_ptr(bf.name) }.to_str().unwrap(); + write!(f, "InvokeBuiltin{} {}", + if *leaf { " leaf" } else { "" }, + // e.g. Code that use `Primitive.cexpr!`. From BUILTIN_INLINE_PREFIX. + if bf_name.starts_with("_bi") { "<inline_expr>" } else { bf_name })?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } + &Insn::EntryPoint { jit_entry_idx: Some(idx) } => write!(f, "EntryPoint JIT({idx})"), + &Insn::EntryPoint { jit_entry_idx: None } => write!(f, "EntryPoint interpreter"), Insn::Return { val } => { write!(f, "Return {val}") } Insn::FixnumAdd { left, right, .. } => { write!(f, "FixnumAdd {left}, {right}") }, Insn::FixnumSub { left, right, .. } => { write!(f, "FixnumSub {left}, {right}") }, Insn::FixnumMult { left, right, .. } => { write!(f, "FixnumMult {left}, {right}") }, Insn::FixnumDiv { left, right, .. } => { write!(f, "FixnumDiv {left}, {right}") }, Insn::FixnumMod { left, right, .. } => { write!(f, "FixnumMod {left}, {right}") }, + Insn::FloatAdd { recv, other, .. } => { write!(f, "FloatAdd {recv}, {other}") }, + Insn::FloatSub { recv, other, .. } => { write!(f, "FloatSub {recv}, {other}") }, + Insn::FloatMul { recv, other, .. } => { write!(f, "FloatMul {recv}, {other}") }, + Insn::FloatDiv { recv, other, .. } => { write!(f, "FloatDiv {recv}, {other}") }, + Insn::FloatToInt { recv, .. } => { write!(f, "FloatToInt {recv}") }, Insn::FixnumEq { left, right, .. } => { write!(f, "FixnumEq {left}, {right}") }, Insn::FixnumNeq { left, right, .. } => { write!(f, "FixnumNeq {left}, {right}") }, Insn::FixnumLt { left, right, .. } => { write!(f, "FixnumLt {left}, {right}") }, @@ -807,17 +2120,75 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::FixnumGe { left, right, .. } => { write!(f, "FixnumGe {left}, {right}") }, Insn::FixnumAnd { left, right, .. } => { write!(f, "FixnumAnd {left}, {right}") }, Insn::FixnumOr { left, right, .. } => { write!(f, "FixnumOr {left}, {right}") }, - Insn::GuardType { val, guard_type, .. } => { write!(f, "GuardType {val}, {}", guard_type.print(self.ptr_map)) }, - Insn::GuardBitEquals { val, expected, .. } => { write!(f, "GuardBitEquals {val}, {}", expected.print(self.ptr_map)) }, + Insn::FixnumXor { left, right, .. } => { write!(f, "FixnumXor {left}, {right}") }, + Insn::IntAnd { left, right } => { write!(f, "IntAnd {left}, {right}") }, + Insn::IntOr { left, right } => { write!(f, "IntOr {left}, {right}") }, + Insn::FixnumLShift { left, right, .. } => { write!(f, "FixnumLShift {left}, {right}") }, + Insn::FixnumRShift { left, right, .. } => { write!(f, "FixnumRShift {left}, {right}") }, + Insn::GuardType { val, guard_type, recompile, .. } => { + write!(f, "GuardType {val}, {}", guard_type.print(self.ptr_map))?; + if recompile.is_some() { + write!(f, " recompile")?; + } + return Ok(()) + }, + Insn::RefineType { val, new_type, .. } => { write!(f, "RefineType {val}, {}", new_type.print(self.ptr_map)) }, + Insn::HasType { val, expected, .. } => { write!(f, "HasType {val}, {}", expected.print(self.ptr_map)) }, + Insn::GuardBitEquals { val, expected, recompile, .. } => { + write!(f, "GuardBitEquals {val}, {}", expected.print(self.ptr_map))?; + if recompile.is_some() { + write!(f, " recompile")?; + } + return Ok(()) + }, + Insn::GuardAnyBitSet { val, mask, mask_name: Some(name), .. } => { write!(f, "GuardAnyBitSet {val}, {name}={}", mask.print(self.ptr_map)) }, + Insn::GuardAnyBitSet { val, mask, .. } => { write!(f, "GuardAnyBitSet {val}, {}", mask.print(self.ptr_map)) }, + Insn::GuardNoBitsSet { val, mask, mask_name: Some(name), .. } => { write!(f, "GuardNoBitsSet {val}, {name}={}", mask.print(self.ptr_map)) }, + Insn::GuardNoBitsSet { val, mask, .. } => { write!(f, "GuardNoBitsSet {val}, {}", mask.print(self.ptr_map)) }, + Insn::GuardLess { left, right, .. } => write!(f, "GuardLess {left}, {right}"), + Insn::GuardGreaterEq { left, right, .. } => write!(f, "GuardGreaterEq {left}, {right}"), + &Insn::GetBlockParam { level, ep_offset, .. } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset) + .map_or(String::new(), |x| format!("{x}, ")); + write!(f, "GetBlockParam {name}l{level}, EP@{ep_offset}") + }, Insn::PatchPoint { invariant, .. } => { write!(f, "PatchPoint {}", invariant.print(self.ptr_map)) }, + Insn::GetConstant { klass, id, allow_nil, .. } => { + write!(f, "GetConstant {klass}, :{}, {allow_nil}", id.contents_lossy()) + } Insn::GetConstantPath { ic, .. } => { write!(f, "GetConstantPath {:p}", self.ptr_map.map_ptr(ic)) }, - Insn::CCall { cfun, args, name, return_type: _, elidable: _ } => { - write!(f, "CCall {}@{:p}", name.contents_lossy(), self.ptr_map.map_ptr(cfun))?; + Insn::IsBlockGiven { lep } => { write!(f, "IsBlockGiven {lep}") }, + Insn::FixnumBitCheck {val, index} => { write!(f, "FixnumBitCheck {val}, {index}") }, + Insn::CCall { cfunc, recv, args, name, owner, return_type: _, elidable: _ } => { + let display_name = if *owner == Qnil { name.contents_lossy().to_string() } else { qualified_method_name(*owner, *name) }; + write!(f, "CCall {recv}, :{}@{:p}", display_name, self.ptr_map.map_ptr(cfunc))?; for arg in args { write!(f, ", {arg}")?; } Ok(()) }, + Insn::CCallWithFrame { cfunc, recv, args, name, cme, block, .. } => { + write!(f, "CCallWithFrame {recv}, :{}@{:p}", qualified_method_name(unsafe { (**cme).owner }, *name), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + match block { + Some(BlockHandler::BlockIseq(blockiseq)) => + write!(f, ", block={:p}", self.ptr_map.map_ptr(blockiseq))?, + Some(BlockHandler::BlockArg) => + write!(f, ", block=&block")?, + None => {} + } + Ok(()) + }, + Insn::CCallVariadic { cfunc, recv, args, name, cme, .. } => { + write!(f, "CCallVariadic {recv}, :{}@{:p}", qualified_method_name(unsafe { (**cme).owner }, *name), self.ptr_map.map_ptr(cfunc))?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + }, + Insn::IncrCounterPtr { .. } => write!(f, "IncrCounterPtr"), Insn::Snapshot { state } => write!(f, "Snapshot {}", state.print(self.ptr_map)), Insn::Defined { op_type, v, .. } => { // op_type (enum defined_type) printing logic from iseq.c. @@ -837,13 +2208,47 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { } Insn::DefinedIvar { self_val, id, .. } => write!(f, "DefinedIvar {self_val}, :{}", id.contents_lossy()), Insn::GetIvar { self_val, id, .. } => write!(f, "GetIvar {self_val}, :{}", id.contents_lossy()), + Insn::CheckMatch { target, pattern, flag, .. } => { + const TYPE_MASK: u32 = 0x03; + const ARRAY_FLAG: u32 = 0x04; + + let match_type = match *flag & TYPE_MASK { + VM_CHECKMATCH_TYPE_WHEN => "WHEN", + VM_CHECKMATCH_TYPE_CASE => "CASE", + VM_CHECKMATCH_TYPE_RESCUE => "RESCUE", + _ => return write!(f, "CheckMatch {target}, {pattern}, {flag}"), + }; + let flag = if *flag & ARRAY_FLAG != 0 { + format!("{match_type}|ARRAY") + } else { + match_type.to_string() + }; + write!(f, "CheckMatch {target}, {pattern}, {flag}") + } + Insn::LoadPC => write!(f, "LoadPC"), + Insn::LoadEC => write!(f, "LoadEC"), + Insn::LoadSP => write!(f, "LoadSP"), + &Insn::GetEP { level } => write!(f, "GetEP {level}"), + Insn::LoadSelf => write!(f, "LoadSelf"), + &Insn::LoadField { recv, id, offset, return_type: _ } => { + write!(f, "LoadField {recv}, :{id}@{:#x}", self.ptr_map.map_offset(offset)) + } + &Insn::StoreField { recv, id, offset, val } => write!(f, "StoreField {recv}, :{id}@{:#x}, {val}", self.ptr_map.map_offset(offset)), + &Insn::WriteBarrier { recv, val } => write!(f, "WriteBarrier {recv}, {val}"), Insn::SetIvar { self_val, id, val, .. } => write!(f, "SetIvar {self_val}, :{}, {val}", id.contents_lossy()), Insn::GetGlobal { id, .. } => write!(f, "GetGlobal :{}", id.contents_lossy()), Insn::SetGlobal { id, val, .. } => write!(f, "SetGlobal :{}, {val}", id.contents_lossy()), - Insn::GetLocal { level, ep_offset } => write!(f, "GetLocal l{level}, EP@{ep_offset}"), - Insn::SetLocal { val, level, ep_offset } => write!(f, "SetLocal l{level}, EP@{ep_offset}, {val}"), + &Insn::IsBlockParamModified { flags } => { + write!(f, "IsBlockParamModified {flags}") + }, + &Insn::SetLocal { val, level, ep_offset } => { + let name = get_local_var_name_for_printer(self.iseq, level, ep_offset).map_or(String::new(), |x| format!("{x}, ")); + write!(f, "SetLocal {name}l{level}, EP@{ep_offset}, {val}") + }, Insn::GetSpecialSymbol { symbol_type, .. } => write!(f, "GetSpecialSymbol {symbol_type:?}"), Insn::GetSpecialNumber { nth, .. } => write!(f, "GetSpecialNumber {nth}"), + Insn::GetClassVar { id, .. } => write!(f, "GetClassVar :{}", id.contents_lossy()), + Insn::SetClassVar { id, val, .. } => write!(f, "SetClassVar :{}, {val}", id.contents_lossy()), Insn::ToArray { val, .. } => write!(f, "ToArray {val}"), Insn::ToNewArray { val, .. } => write!(f, "ToNewArray {val}"), Insn::ArrayExtend { left, right, .. } => write!(f, "ArrayExtend {left}, {right}"), @@ -851,9 +2256,15 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::ObjToString { val, .. } => { write!(f, "ObjToString {val}") }, Insn::StringIntern { val, .. } => { write!(f, "StringIntern {val}") }, Insn::AnyToString { val, str, .. } => { write!(f, "AnyToString {val}, str: {str}") }, - Insn::SideExit { reason, .. } => write!(f, "SideExit {reason}"), - Insn::PutSpecialObject { value_type } => write!(f, "PutSpecialObject {value_type}"), - Insn::Throw { throw_state, val } => { + Insn::SideExit { reason, recompile, .. } => { + if recompile.is_some() { + write!(f, "SideExit {reason} recompile") + } else { + write!(f, "SideExit {reason}") + } + } + Insn::PutSpecialObject { value_type, .. } => write!(f, "PutSpecialObject {value_type}"), + Insn::Throw { throw_state, val, .. } => { write!(f, "Throw ")?; match throw_state & VM_THROW_STATE_MASK { RUBY_TAG_NONE => write!(f, "TAG_NONE"), @@ -874,13 +2285,16 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { } Insn::IncrCounter(counter) => write!(f, "IncrCounter {counter:?}"), Insn::CheckInterrupts { .. } => write!(f, "CheckInterrupts"), + Insn::IsA { val, class } => write!(f, "IsA {val}, {class}"), + Insn::BreakPoint => write!(f, "BreakPoint"), + Insn::Unreachable => write!(f, "Unreachable"), } } } impl std::fmt::Display for Insn { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - self.print(&PtrPrintMap::identity()).fmt(f) + self.print(&PtrPrintMap::identity(), None).fmt(f) } } @@ -908,7 +2322,7 @@ impl Block { /// Pretty printer for [`Function`]. pub struct FunctionPrinter<'a> { fun: &'a Function, - display_snapshot: bool, + display_snapshot_and_tp_patchpoints: bool, ptr_map: PtrPrintMap, } @@ -918,32 +2332,16 @@ impl<'a> FunctionPrinter<'a> { if cfg!(test) { ptr_map.map_ptrs = true; } - Self { fun, display_snapshot: false, ptr_map } + Self { fun, display_snapshot_and_tp_patchpoints: false, ptr_map } } pub fn with_snapshot(fun: &'a Function) -> FunctionPrinter<'a> { let mut printer = Self::without_snapshot(fun); - printer.display_snapshot = true; + printer.display_snapshot_and_tp_patchpoints = true; printer } } -/// Pretty printer for [`Function`]. -pub struct FunctionGraphvizPrinter<'a> { - fun: &'a Function, - ptr_map: PtrPrintMap, -} - -impl<'a> FunctionGraphvizPrinter<'a> { - pub fn new(fun: &'a Function) -> Self { - let mut ptr_map = PtrPrintMap::identity(); - if cfg!(test) { - ptr_map.map_ptrs = true; - } - Self { fun, ptr_map } - } -} - /// Union-Find (Disjoint-Set) is a data structure for managing disjoint sets that has an interface /// of two operations: /// @@ -983,7 +2381,7 @@ impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { /// Private. Return the internal representation of the forwarding pointer for a given element. fn at(&self, idx: T) -> Option<T> { - self.forwarded.get(idx.into()).map(|x| *x).flatten() + self.forwarded.get(idx.into()).copied().flatten() } /// Private. Set the internal representation of the forwarding pointer for the given element @@ -992,7 +2390,9 @@ impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { if idx.into() >= self.forwarded.len() { self.forwarded.resize(idx.into()+1, None); } - self.forwarded[idx.into()] = Some(value); + if idx != value { + self.forwarded[idx.into()] = Some(value); + } } /// Find the set representative for `insn`. Perform path compression at the same time to speed @@ -1021,7 +2421,10 @@ impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { loop { match self.at(result) { None => return result, - Some(insn) => result = insn, + Some(insn) => { + assert!(result != insn, "cycle detected"); + result = insn; + } } } } @@ -1034,7 +2437,7 @@ impl<T: Copy + Into<usize> + PartialEq> UnionFind<T> { } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ValidationError { BlockHasNoTerminator(BlockId), // The terminator and its actual position @@ -1046,16 +2449,119 @@ pub enum ValidationError { OperandNotDefined(BlockId, InsnId, InsnId), /// The offending block and instruction DuplicateInstruction(BlockId, InsnId), + /// The offending instruction, its operand, expected type string, actual type string + MismatchedOperandType(InsnId, InsnId, String, String), + MiscValidationError(InsnId, String), } -fn can_direct_send(iseq: *const rb_iseq_t) -> bool { - if unsafe { rb_get_iseq_flags_has_rest(iseq) } { false } - else if unsafe { rb_get_iseq_flags_has_opt(iseq) } { false } - else if unsafe { rb_get_iseq_flags_has_kw(iseq) } { false } - else if unsafe { rb_get_iseq_flags_has_kwrest(iseq) } { false } - else if unsafe { rb_get_iseq_flags_has_block(iseq) } { false } - else if unsafe { rb_get_iseq_flags_forwardable(iseq) } { false } - else { true } +/// Check if we can do a direct send to the given iseq with the given args. +fn can_direct_send(function: &mut Function, block: BlockId, iseq: *const rb_iseq_t, ci: *const rb_callinfo, send_insn: InsnId, args: &[InsnId], has_block: bool) -> bool { + let mut can_send = true; + let mut count_failure = |counter| { + can_send = false; + function.count(block, counter); + }; + let params = unsafe { iseq.params() }; + + let callee_has_block_param = 0 != params.flags.has_block(); + let caller_passes_block_arg = (unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_BLOCKARG) != 0; + + use Counter::*; + if 0 != params.flags.has_rest() { count_failure(complex_arg_pass_param_rest) } + if 0 != params.flags.forwardable() { count_failure(complex_arg_pass_param_forwardable) } + if callee_has_block_param && caller_passes_block_arg + { count_failure(complex_arg_pass_param_block) } + if 0 != params.flags.has_kwrest() { count_failure(complex_arg_pass_param_kwrest) } + + // If the caller passes a block (literal or &block), we need to fall back to the + // interpreter for two cases it handles that we don't: + // 1. Methods with &nil reject blocks with ArgumentError + // 2. Methods that don't use blocks emit "unused block" warnings + let caller_passes_block = has_block || caller_passes_block_arg; + if caller_passes_block && 0 != params.flags.accepts_no_block() + { count_failure(complex_arg_pass_accepts_no_block) } + if caller_passes_block && 0 == params.flags.use_block() + { count_failure(complex_arg_pass_does_not_use_block) } + + if !can_send { + function.set_dynamic_send_reason(send_insn, ComplexArgPass); + return false; + } + + let lead_num = params.lead_num; + let opt_num = params.opt_num; + let post_num = params.post_num; + let keyword = params.keyword; + let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } }; + let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } }; + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let caller_positional = match args.len().checked_sub(caller_kw_count) { + Some(count) => count, + None => { + function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); + return false; + } + }; + + let positional_ok = c_int::try_from(caller_positional) + .as_ref() + .map(|argc| (lead_num + post_num..=lead_num + opt_num + post_num).contains(argc)) + .unwrap_or(false); + let keyword_ok = c_int::try_from(caller_kw_count) + .as_ref() + .map(|argc| (kw_req_num..=kw_total_num).contains(argc)) + .unwrap_or(false); + if !positional_ok || !keyword_ok { + function.set_dynamic_send_reason(send_insn, ArgcParamMismatch); + return false + } + + // asm.ccall() doesn't support 6+ args. Compute the final argc after keyword setup: + // final_argc = caller's positional args + callee's total keywords (all kw slots are filled). + // Right now, the JIT entrypoint accepts the block as an param + // We may remove it, remove the block_arg addition to match + // See: https://github.com/ruby/ruby/pull/15911#discussion_r2710544982 + let block_arg = if 0 != params.flags.has_block() { 1 } else { 0 }; + let final_argc = caller_positional + kw_total_num as usize + block_arg; + if final_argc + 1 > C_ARG_OPNDS.len() { // +1 for self + function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); + return false; + } + + // IseqCall stores num_optionals_passed and argc as u16 + if u16::try_from(args.len()).is_err() { + function.set_dynamic_send_reason(send_insn, TooManyArgsForLir); + return false; + } + + can_send +} + +/// Policy that controls how optimization passes generate code. +/// Determined at compile time based on the ISEQ's compilation history. +#[derive(Debug)] +struct CompilePolicy { + /// When true, optimization passes should avoid generating guards that + /// side-exit, and instead use fallback paths (e.g. C calls) on mismatch. + /// Set when this is the final version of an ISEQ after recompilation. + no_side_exits: bool, +} + +impl CompilePolicy { + fn new(iseq: *const rb_iseq_t) -> Self { + // When a previous version was invalidated and we've reached the version + // limit, avoid speculative optimizations that may side-exit. + let no_side_exits = if iseq.is_null() { + false + } else { + let payload = get_or_create_iseq_payload(iseq); + payload.versions.iter().any( + |v| unsafe { v.as_ref() }.is_invalidated() + ) && payload.versions.len() + 1 >= max_iseq_versions() + }; + Self { no_side_exits } + } } /// A [`Function`], which is analogous to a Ruby ISeq, is a control-flow graph of [`Block`]s @@ -1064,33 +2570,141 @@ fn can_direct_send(iseq: *const rb_iseq_t) -> bool { pub struct Function { // ISEQ this function refers to iseq: *const rb_iseq_t, - // The types for the parameters of this function + /// Whether previously, a function for this ISEQ was invalidated due to + /// singleton class creation (violation of NoSingletonClass invariant). + was_invalidated_for_singleton_class_creation: bool, + /// Whether `self` is guaranteed to be a heap (non-immediate) object. When set, + /// the `self`-producing instructions (`LoadSelf` and the `SelfParam` `LoadArg`) + /// are typed `HeapBasicObject` instead of `BasicObject`. Sourced from + /// `IseqPayload::self_is_heap_object`. + self_is_heap_object: bool, + /// Controls code generation strategy for optimization passes. + policy: CompilePolicy, + /// The types for the parameters of this function. They are copied to the type + /// of entry block params after infer_types() fills Empty to all insn_types. param_types: Vec<Type>, - // TODO: get method name and source location from the ISEQ - insns: Vec<Insn>, union_find: std::cell::RefCell<UnionFind<InsnId>>, insn_types: Vec<Type>, blocks: Vec<Block>, + /// Superblock that targets all entry blocks. The sole root for RPO/dominator computation. + pub entries_block: BlockId, + /// Entry block for the interpreter entry_block: BlockId, + /// Entry block for JIT-to-JIT calls. Length will be `opt_num+1`, for callers + /// fulfilling `(0..=opt_num)` optional parameters. + jit_entry_blocks: Vec<BlockId>, profiles: Option<ProfileOracle>, } +/// The kind of a value an ISEQ returns +enum IseqReturn { + Value(VALUE), + LocalVariable(u32), + Receiver, + // Builtin descriptor and return type (if known) + InvokeLeafBuiltin(rb_builtin_function, Option<Type>), +} + +unsafe extern "C" { + fn rb_simple_iseq_p(iseq: IseqPtr) -> bool; +} + +/// Return the ISEQ's return value if it consists of one simple instruction and leave. +fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option<InsnId>, ci_flags: u32) -> Option<IseqReturn> { + // Expect only two instructions and one possible operand + // NOTE: If an ISEQ has an optional keyword parameter with a default value that requires + // computation, the ISEQ will always have more than two instructions and won't be inlined. + + // Get the first two instructions + let first_insn = iseq_opcode_at_idx(iseq, 0); + let second_insn = iseq_opcode_at_idx(iseq, insn_len(first_insn as usize)); + + // Extract the return value if known + if second_insn != YARVINSN_leave { + return None; + } + match first_insn { + YARVINSN_getlocal_WC_0 => { + // Accept only cases where only positional arguments are used by both the callee and the caller. + // Keyword arguments may be specified by the callee or the caller but not used. + if captured_opnd.is_some() + // Equivalent to `VM_CALL_ARGS_SIMPLE - VM_CALL_KWARG - has_block_iseq` + || ci_flags & ( + VM_CALL_ARGS_SPLAT + | VM_CALL_KW_SPLAT + | VM_CALL_ARGS_BLOCKARG + | VM_CALL_FORWARDING + ) != 0 + { + return None; + } + + let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); + let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + + // Only inline if the local is a parameter (not a method-defined local) as we are indexing args. + let param_size = unsafe { rb_get_iseq_body_param_size(iseq) } as usize; + if local_idx >= param_size { + return None; + } + + if unsafe { rb_simple_iseq_p(iseq) } { + return Some(IseqReturn::LocalVariable(local_idx.try_into().unwrap())); + } + + // TODO(max): Support only_kwparam case where the local_idx is a positional parameter + + None + } + YARVINSN_putnil => Some(IseqReturn::Value(Qnil)), + YARVINSN_putobject => Some(IseqReturn::Value(unsafe { *rb_iseq_pc_at_idx(iseq, 1) })), + YARVINSN_putobject_INT2FIX_0_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(0))), + YARVINSN_putobject_INT2FIX_1_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(1))), + // We don't support invokeblock for now. Such ISEQs are likely not used by blocks anyway. + YARVINSN_putself if captured_opnd.is_none() => Some(IseqReturn::Receiver), + YARVINSN_opt_invokebuiltin_delegate_leave => { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; + let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let argc = bf.argc as usize; + if argc != 0 { return None; } + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + if !leaf { return None; } + // Check if this builtin is annotated + let return_type = ZJITState::get_method_annotations() + .get_builtin_properties(&bf) + .map(|props| props.return_type); + Some(IseqReturn::InvokeLeafBuiltin(bf, return_type)) + } + _ => None, + } +} + impl Function { fn new(iseq: *const rb_iseq_t) -> Function { Function { iseq, + was_invalidated_for_singleton_class_creation: false, + self_is_heap_object: false, + policy: CompilePolicy::new(iseq), insns: vec![], insn_types: vec![], union_find: UnionFind::new().into(), - blocks: vec![Block::default()], - entry_block: BlockId(0), + blocks: vec![Block::default(), Block::default()], + entries_block: BlockId(0), + entry_block: BlockId(1), + jit_entry_blocks: vec![], param_types: vec![], profiles: None, } } + pub fn iseq(&self) -> *const rb_iseq_t { + self.iseq + } + // Add an instruction to the function without adding it to any block fn new_insn(&mut self, insn: Insn) -> InsnId { let id = InsnId(self.insns.len()); @@ -1104,8 +2718,8 @@ impl Function { } // Add an instruction to an SSA block - fn push_insn(&mut self, block: BlockId, insn: Insn) -> InsnId { - let is_param = matches!(insn, Insn::Param { .. }); + pub fn push_insn(&mut self, block: BlockId, insn: Insn) -> InsnId { + let is_param = matches!(insn, Insn::Param); let id = self.new_insn(insn); if is_param { self.blocks[block.0].params.push(id); @@ -1115,6 +2729,10 @@ impl Function { id } + pub fn push_comment(&mut self, block: BlockId, message: String) -> InsnId { + self.push_insn(block, Insn::Comment { message }) + } + // Add an instruction to an SSA block fn push_insn_id(&mut self, block: BlockId, insn_id: InsnId) -> InsnId { self.blocks[block.0].insns.push(insn_id); @@ -1136,22 +2754,102 @@ impl Function { fn new_block(&mut self, insn_idx: u32) -> BlockId { let id = BlockId(self.blocks.len()); - let mut block = Block::default(); - block.insn_idx = insn_idx; + let block = Block { + insn_idx, + .. Block::default() + }; self.blocks.push(block); id } + fn remove_block(&mut self, block_id: BlockId) { + if BlockId(self.blocks.len() - 1) != block_id { + panic!("Can only remove the last block"); + } + self.blocks.pop(); + } + + fn successors(&self, block: BlockId) -> Vec<BlockId> { + let insns = &self.blocks[block.0].insns; + let last = self.find(*insns.last().unwrap()); + match last { + Insn::CondBranch { if_true, if_false, .. } => vec![if_true.target, if_false.target], + Insn::Jump(edge) => vec![edge.target], + Insn::Entries { targets } => targets, + Insn::Unreachable | Insn::Return { .. } | Insn::SideExit { .. } | Insn::Throw { .. } => vec![], + // Blocks that don't end with terminators are technically errors, + // every block in the CFG should end with a terminator. But we + // want to be able to iterate over poorly constructed CFG when + // debugging, so we'll return an empty vec. The validation + // routines check for terminators, so we should catch CFG errors there. + _ => vec![] + } + } + /// Return a reference to the Block at the given index. pub fn block(&self, block_id: BlockId) -> &Block { &self.blocks[block_id.0] } + /// Return a reference to the entry block. + pub fn entry_block(&self) -> &Block { + &self.blocks[self.entry_block.0] + } + /// Return the number of blocks pub fn num_blocks(&self) -> usize { self.blocks.len() } + pub fn assume_single_ractor_mode(&mut self, block: BlockId, state: InsnId) -> bool { + if unsafe { rb_jit_multi_ractor_p() } { + false + } else { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::SingleRactorMode, state }); + true + } + } + + /// Assume that only the root box is active, so we can safely read from the prime classext. + /// Returns true if safe to assume so and emits a PatchPoint. + pub fn assume_root_box(&mut self, block: BlockId, state: InsnId) -> bool { + if invariants::non_root_box_created() { + false + } else { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::RootBoxOnly, state }); + true + } + } + + /// Assume that objects of a given class will have no singleton class. + /// Returns true if safe to assume so and emits a PatchPoint. + /// Returns false if we've already seen a singleton class for this class, + /// to avoid an invalidation loop. + pub fn assume_no_singleton_classes(&mut self, block: BlockId, klass: VALUE, state: InsnId) -> bool { + if !klass.instance_can_have_singleton_class() { + // This class can never have a singleton class, so no patchpoint needed. + return true; + } + if klass.is_singleton_class() { + // When a value has a singleton class, its effective class can't change anymore. + // No patchpoint needed. + return true; + } + if self.was_invalidated_for_singleton_class_creation && invariants::has_singleton_class_of(klass) { + // A previous compilation of this ISEQ was invalidated for singleton class + // creation. Avoid repeating the invalidation. + return false; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass }, state }); + true + } + + pub fn count(&mut self, block: BlockId, counter: Counter) { + if get_option!(stats) { + self.push_insn(block, Insn::IncrCounter(counter)); + } + } + /// Return a copy of the instruction where the instruction and its operands have been read from /// the union-find table (to find the current most-optimized version of this instruction). See /// [`UnionFind`] for more. @@ -1177,134 +2875,36 @@ impl Function { } }; } - macro_rules! find_vec { - ( $x:expr ) => { - { - $x.iter().map(|arg| find!(*arg)).collect() - } - }; - } - macro_rules! find_branch_edge { - ( $edge:ident ) => { - { - BranchEdge { - target: $edge.target, - args: find_vec!($edge.args), - } - } - }; - } let insn_id = find!(insn_id); + let mut result = self.insns[insn_id.0].clone(); + result.for_each_operand_mut(&mut |operand: &mut InsnId| { + *operand = find!(*operand); + }); + result + } + + /// Update DynamicSendReason for the instruction at insn_id + fn set_dynamic_send_reason(&mut self, insn_id: InsnId, dynamic_send_reason: SendFallbackReason) { use Insn::*; - match &self.insns[insn_id.0] { - result@(Const {..} - | Param {..} - | GetConstantPath {..} - | PatchPoint {..} - | PutSpecialObject {..} - | GetGlobal {..} - | GetLocal {..} - | SideExit {..} - | IncrCounter(_)) => result.clone(), - &Snapshot { state: FrameState { iseq, insn_idx, pc, ref stack, ref locals } } => - Snapshot { - state: FrameState { - iseq, - insn_idx, - pc, - stack: find_vec!(stack), - locals: find_vec!(locals), - } - }, - &Return { val } => Return { val: find!(val) }, - &Throw { throw_state, val } => Throw { throw_state, val: find!(val) }, - &StringCopy { val, chilled, state } => StringCopy { val: find!(val), chilled, state }, - &StringIntern { val, state } => StringIntern { val: find!(val), state: find!(state) }, - &StringConcat { ref strings, state } => StringConcat { strings: find_vec!(strings), state: find!(state) }, - &ToRegexp { opt, ref values, state } => ToRegexp { opt, values: find_vec!(values), state }, - &Test { val } => Test { val: find!(val) }, - &IsNil { val } => IsNil { val: find!(val) }, - &Jump(ref target) => Jump(find_branch_edge!(target)), - &IfTrue { val, ref target } => IfTrue { val: find!(val), target: find_branch_edge!(target) }, - &IfFalse { val, ref target } => IfFalse { val: find!(val), target: find_branch_edge!(target) }, - &GuardType { val, guard_type, state } => GuardType { val: find!(val), guard_type: guard_type, state }, - &GuardBitEquals { val, expected, state } => GuardBitEquals { val: find!(val), expected: expected, state }, - &FixnumAdd { left, right, state } => FixnumAdd { left: find!(left), right: find!(right), state }, - &FixnumSub { left, right, state } => FixnumSub { left: find!(left), right: find!(right), state }, - &FixnumMult { left, right, state } => FixnumMult { left: find!(left), right: find!(right), state }, - &FixnumDiv { left, right, state } => FixnumDiv { left: find!(left), right: find!(right), state }, - &FixnumMod { left, right, state } => FixnumMod { left: find!(left), right: find!(right), state }, - &FixnumNeq { left, right } => FixnumNeq { left: find!(left), right: find!(right) }, - &FixnumEq { left, right } => FixnumEq { left: find!(left), right: find!(right) }, - &FixnumGt { left, right } => FixnumGt { left: find!(left), right: find!(right) }, - &FixnumGe { left, right } => FixnumGe { left: find!(left), right: find!(right) }, - &FixnumLt { left, right } => FixnumLt { left: find!(left), right: find!(right) }, - &FixnumLe { left, right } => FixnumLe { left: find!(left), right: find!(right) }, - &FixnumAnd { left, right } => FixnumAnd { left: find!(left), right: find!(right) }, - &FixnumOr { left, right } => FixnumOr { left: find!(left), right: find!(right) }, - &ObjToString { val, cd, state } => ObjToString { - val: find!(val), - cd: cd, - state, - }, - &AnyToString { val, str, state } => AnyToString { - val: find!(val), - str: find!(str), - state, - }, - &SendWithoutBlock { self_val, cd, ref args, state } => SendWithoutBlock { - self_val: find!(self_val), - cd: cd, - args: find_vec!(args), - state, - }, - &SendWithoutBlockDirect { self_val, cd, cme, iseq, ref args, state } => SendWithoutBlockDirect { - self_val: find!(self_val), - cd: cd, - cme: cme, - iseq: iseq, - args: find_vec!(args), - state, - }, - &Send { self_val, cd, blockiseq, ref args, state } => Send { - self_val: find!(self_val), - cd: cd, - blockiseq: blockiseq, - args: find_vec!(args), - state, - }, - &InvokeBuiltin { bf, ref args, state, return_type } => InvokeBuiltin { bf, args: find_vec!(args), state, return_type }, - &ArrayDup { val, state } => ArrayDup { val: find!(val), state }, - &HashDup { val, state } => HashDup { val: find!(val), state }, - &CCall { cfun, ref args, name, return_type, elidable } => CCall { cfun, args: find_vec!(args), name, return_type, elidable }, - &Defined { op_type, obj, pushval, v, state } => Defined { op_type, obj, pushval, v: find!(v), state: find!(state) }, - &DefinedIvar { self_val, pushval, id, state } => DefinedIvar { self_val: find!(self_val), pushval, id, state }, - &NewArray { ref elements, state } => NewArray { elements: find_vec!(elements), state: find!(state) }, - &NewHash { ref elements, state } => { - let mut found_elements = vec![]; - for &(key, value) in elements { - found_elements.push((find!(key), find!(value))); - } - NewHash { elements: found_elements, state: find!(state) } - } - &NewRange { low, high, flag, state } => NewRange { low: find!(low), high: find!(high), flag, state: find!(state) }, - &ArrayMax { ref elements, state } => ArrayMax { elements: find_vec!(elements), state: find!(state) }, - &SetGlobal { id, val, state } => SetGlobal { id, val: find!(val), state }, - &GetIvar { self_val, id, state } => GetIvar { self_val: find!(self_val), id, state }, - &SetIvar { self_val, id, val, state } => SetIvar { self_val: find!(self_val), id, val: find!(val), state }, - &SetLocal { val, ep_offset, level } => SetLocal { val: find!(val), ep_offset, level }, - &GetSpecialSymbol { symbol_type, state } => GetSpecialSymbol { symbol_type, state }, - &GetSpecialNumber { nth, state } => GetSpecialNumber { nth, state }, - &ToArray { val, state } => ToArray { val: find!(val), state }, - &ToNewArray { val, state } => ToNewArray { val: find!(val), state }, - &ArrayExtend { left, right, state } => ArrayExtend { left: find!(left), right: find!(right), state }, - &ArrayPush { array, val, state } => ArrayPush { array: find!(array), val: find!(val), state }, - &CheckInterrupts { state } => CheckInterrupts { state }, + // Always set the reason: convert_no_profile_sends depends on it to identify + // sends that should be converted to side exits for exit-based recompilation. + match self.insns.get_mut(insn_id.0).unwrap() { + Send { reason, .. } + | SendForward { reason, .. } + | InvokeSuper { reason, .. } + | InvokeSuperForward { reason, .. } + | InvokeBlock { reason, .. } + => *reason = dynamic_send_reason, + _ => unreachable!("unexpected instruction {} at {insn_id}", self.find(insn_id)) } } /// Replace `insn` with the new instruction `replacement`, which will get appended to `insns`. fn make_equal_to(&mut self, insn: InsnId, replacement: InsnId) { + assert!(self.insns[insn.0].has_output(), + "Don't use make_equal_to for instruction with no output"); + assert!(self.insns[replacement.0].has_output(), + "Can't replace instruction that has output with instruction that has no output"); // Don't push it to the block self.union_find.borrow_mut().make_equal_to(insn, replacement); } @@ -1315,20 +2915,24 @@ impl Function { } /// Check if the type of `insn` is a subtype of `ty`. - fn is_a(&self, insn: InsnId, ty: Type) -> bool { + pub fn is_a(&self, insn: InsnId, ty: Type) -> bool { self.type_of(insn).is_subtype(ty) } fn infer_type(&self, insn: InsnId) -> Type { assert!(self.insns[insn.0].has_output()); match &self.insns[insn.0] { - Insn::Param { .. } => unimplemented!("params should not be present in block.insns"), - Insn::SetGlobal { .. } | Insn::Jump(_) - | Insn::IfTrue { .. } | Insn::IfFalse { .. } | Insn::Return { .. } | Insn::Throw { .. } - | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::ArrayExtend { .. } - | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetLocal { .. } | Insn::IncrCounter(_) - | Insn::CheckInterrupts { .. } => - panic!("Cannot infer type of instruction with no output: {}", self.insns[insn.0]), + Insn::Param => unimplemented!("params should not be present in block.insns"), + Insn::LoadArg { val_type, .. } => *val_type, + Insn::SetGlobal { .. } | Insn::Jump(_) | Insn::Entries { .. } | Insn::EntryPoint { .. } + | Insn::Comment { .. } + | Insn::CondBranch { .. } | Insn::Return { .. } | Insn::Throw { .. } + | Insn::PatchPoint { .. } | Insn::SetIvar { .. } | Insn::SetClassVar { .. } | Insn::ArrayExtend { .. } + | Insn::ArrayPush { .. } | Insn::SideExit { .. } | Insn::SetLocal { .. } + | Insn::IncrCounter(_) | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } | Insn::BreakPoint | Insn::Unreachable + | Insn::StoreField { .. } | Insn::WriteBarrier { .. } | Insn::HashAset { .. } | Insn::ArrayAset { .. } => + panic!("Cannot infer type of instruction with no output: {}. See Insn::has_output().", self.insns[insn.0]), Insn::Const { val: Const::Value(val) } => Type::from_value(*val), Insn::Const { val: Const::CBool(val) } => Type::from_cbool(*val), Insn::Const { val: Const::CInt8(val) } => Type::from_cint(types::CInt8, *val as i64), @@ -1338,32 +2942,72 @@ impl Function { Insn::Const { val: Const::CUInt8(val) } => Type::from_cint(types::CUInt8, *val as i64), Insn::Const { val: Const::CUInt16(val) } => Type::from_cint(types::CUInt16, *val as i64), Insn::Const { val: Const::CUInt32(val) } => Type::from_cint(types::CUInt32, *val as i64), + Insn::Const { val: Const::CAttrIndex(val) } => Type::from_cint(types::CAttrIndex, *val as i64), + Insn::Const { val: Const::CShape(val) } => Type::from_cint(types::CShape, val.0 as i64), Insn::Const { val: Const::CUInt64(val) } => Type::from_cint(types::CUInt64, *val as i64), - Insn::Const { val: Const::CPtr(val) } => Type::from_cint(types::CPtr, *val as i64), + Insn::Const { val: Const::CPtr(val) } => Type::from_cptr(*val), Insn::Const { val: Const::CDouble(val) } => Type::from_double(*val), Insn::Test { val } if self.type_of(*val).is_known_falsy() => Type::from_cbool(false), Insn::Test { val } if self.type_of(*val).is_known_truthy() => Type::from_cbool(true), Insn::Test { .. } => types::CBool, - Insn::IsNil { val } if self.is_a(*val, types::NilClass) => Type::from_cbool(true), - Insn::IsNil { val } if !self.type_of(*val).could_be(types::NilClass) => Type::from_cbool(false), - Insn::IsNil { .. } => types::CBool, + Insn::IsMethodCfunc { .. } => types::CBool, + Insn::IsBitEqual { .. } => types::CBool, + Insn::IsBitNotEqual { .. } => types::CBool, + Insn::BoxBool { .. } => types::BoolExact, + Insn::BoxFixnum { .. } => types::Fixnum, + Insn::UnboxFixnum { val } => self + .type_of(*val) + .fixnum_value() + .map_or(types::CInt64, |fixnum| Type::from_cint(types::CInt64, fixnum)), + Insn::FixnumAref { .. } => types::Fixnum, Insn::StringCopy { .. } => types::StringExact, Insn::StringIntern { .. } => types::Symbol, Insn::StringConcat { .. } => types::StringExact, + Insn::StringGetbyte { .. } => types::Fixnum, + Insn::StringSetbyteFixnum { .. } => types::Fixnum, + Insn::StringAppend { .. } => types::StringExact, + Insn::StringAppendCodepoint { .. } => types::StringExact, + Insn::StringEqual { .. } => types::BoolExact, Insn::ToRegexp { .. } => types::RegexpExact, Insn::NewArray { .. } => types::ArrayExact, Insn::ArrayDup { .. } => types::ArrayExact, + Insn::ArrayAref { .. } => types::BasicObject, + Insn::ArrayPop { .. } => types::BasicObject, + Insn::ArrayLength { .. } => types::CInt64, + Insn::AdjustBounds { .. } => types::CInt64, + Insn::HashAref { .. } => types::BasicObject, Insn::NewHash { .. } => types::HashExact, Insn::HashDup { .. } => types::HashExact, Insn::NewRange { .. } => types::RangeExact, + Insn::NewRangeFixnum { .. } => types::RangeExact, + Insn::ObjectAlloc { .. } => types::HeapBasicObject, + Insn::ObjectAllocClass { class, .. } => Type::from_class(*class), + &Insn::CCallWithFrame { return_type, .. } => return_type, Insn::CCall { return_type, .. } => *return_type, + &Insn::CCallVariadic { return_type, .. } => return_type, + Insn::CheckMatch { .. } => types::BasicObject, Insn::GuardType { val, guard_type, .. } => self.type_of(*val).intersection(*guard_type), - Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_value(*expected)), + Insn::RefineType { val, new_type, .. } => self.type_of(*val).intersection(*new_type), + &Insn::HasType { val, expected } if self.is_a(val, expected) => Type::from_cbool(true), + &Insn::HasType { val, expected } if !self.type_of(val).could_be(expected) => Type::from_cbool(false), + Insn::HasType { .. } => types::CBool, + Insn::GuardBitEquals { val, expected, .. } => self.type_of(*val).intersection(Type::from_const(*expected)), + Insn::GuardAnyBitSet { val, .. } => self.type_of(*val), + Insn::GuardNoBitsSet { val, .. } => self.type_of(*val), + Insn::GuardLess { left, .. } => self.type_of(*left), + Insn::GuardGreaterEq { left, .. } => self.type_of(*left), Insn::FixnumAdd { .. } => types::Fixnum, Insn::FixnumSub { .. } => types::Fixnum, Insn::FixnumMult { .. } => types::Fixnum, - Insn::FixnumDiv { .. } => types::Fixnum, + // FIXNUM_MIN / -1 overflows to a Bignum, so the result is Integer, not Fixnum. + // Downstream Fixnum ops insert their own GuardType(Fixnum) + Insn::FixnumDiv { .. } => types::Integer, Insn::FixnumMod { .. } => types::Fixnum, + Insn::FloatAdd { .. } => types::Float, + Insn::FloatSub { .. } => types::Float, + Insn::FloatMul { .. } => types::Float, + Insn::FloatDiv { .. } => types::Float, + Insn::FloatToInt { .. } => types::Integer, Insn::FixnumEq { .. } => types::BoolExact, Insn::FixnumNeq { .. } => types::BoolExact, Insn::FixnumLt { .. } => types::BoolExact, @@ -1372,27 +3016,88 @@ impl Function { Insn::FixnumGe { .. } => types::BoolExact, Insn::FixnumAnd { .. } => types::Fixnum, Insn::FixnumOr { .. } => types::Fixnum, + Insn::FixnumXor { .. } => types::Fixnum, + Insn::IntAnd { .. } => types::CInt64, + Insn::IntOr { left, .. } => self.type_of(*left).unspecialized(), + Insn::FixnumLShift { .. } => types::Fixnum, + Insn::FixnumRShift { .. } => types::Fixnum, Insn::PutSpecialObject { .. } => types::BasicObject, - Insn::SendWithoutBlock { .. } => types::BasicObject, - Insn::SendWithoutBlockDirect { .. } => types::BasicObject, + Insn::SendDirect { .. } => types::BasicObject, Insn::Send { .. } => types::BasicObject, + Insn::SendForward { .. } => types::BasicObject, + Insn::InvokeSuper { .. } => types::BasicObject, + Insn::InvokeSuperForward { .. } => types::BasicObject, + Insn::InvokeBlock { .. } => types::BasicObject, + Insn::InvokeBlockIfunc { .. } => types::BasicObject, + Insn::InvokeProc { .. } => types::BasicObject, Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), - Insn::DefinedIvar { .. } => types::BasicObject, + Insn::DefinedIvar { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), + Insn::GetConstant { .. } => types::BasicObject, Insn::GetConstantPath { .. } => types::BasicObject, + Insn::IsBlockGiven { .. } => types::BoolExact, + Insn::FixnumBitCheck { .. } => types::BoolExact, Insn::ArrayMax { .. } => types::BasicObject, + Insn::ArrayMin { .. } => types::BasicObject, + Insn::ArrayInclude { .. } => types::BoolExact, + Insn::ArrayPackBuffer { .. } => types::String, + Insn::DupArrayInclude { .. } => types::BoolExact, + Insn::ArrayHash { .. } => types::Fixnum, Insn::GetGlobal { .. } => types::BasicObject, Insn::GetIvar { .. } => types::BasicObject, - Insn::GetSpecialSymbol { .. } => types::BasicObject, - Insn::GetSpecialNumber { .. } => types::BasicObject, + Insn::LoadPC => types::CPtr, + Insn::LoadSP => types::CPtr, + Insn::LoadEC => types::CPtr, + Insn::GetEP { .. } => types::CPtr, + Insn::LoadSelf => if self.self_is_heap_object { types::HeapBasicObject } else { types::BasicObject }, + &Insn::LoadField { return_type, .. } => return_type, + Insn::GetSpecialSymbol { .. } => types::StringExact.union(types::NilClass), + Insn::GetSpecialNumber { .. } => types::StringExact.union(types::NilClass), + Insn::GetClassVar { .. } => types::BasicObject, Insn::ToNewArray { .. } => types::ArrayExact, Insn::ToArray { .. } => types::ArrayExact, Insn::ObjToString { .. } => types::BasicObject, Insn::AnyToString { .. } => types::String, - Insn::GetLocal { .. } => types::BasicObject, + Insn::IsBlockParamModified { .. } => types::CBool, + Insn::GetBlockParam { .. } => types::BasicObject, // The type of Snapshot doesn't really matter; it's never materialized. It's used only // as a reference for FrameState, which we use to generate side-exit code. Insn::Snapshot { .. } => types::Any, + Insn::IsA { .. } => types::BoolExact, + } + } + + /// Set self.param_types. They are copied to the param types of jit_entry_blocks. + fn set_param_types(&mut self) { + let iseq = self.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + self.param_types.push(types::BasicObject); // self + for local_idx in 0..param_size { + let param_type = if Some(local_idx as i32) == rest_param_idx { + types::ArrayExact // Rest parameters are always ArrayExact + } else { + types::BasicObject + }; + self.param_types.push(param_type); + } + } + + /// Copy self.param_types to the param types of jit_entry_blocks. + fn copy_param_types(&mut self) { + for jit_entry_block in self.jit_entry_blocks.iter() { + let entry_params = self.blocks[jit_entry_block.0].params.iter(); + let param_types = self.param_types.iter(); + assert!( + param_types.len() >= entry_params.len(), + "param types should be initialized before type inference", + ); + for (param, param_type) in std::iter::zip(entry_params, param_types) { + // We know that function parameters are BasicObject or some subclass + self.insn_types[param.0] = *param_type; + } } } @@ -1400,65 +3105,84 @@ impl Function { // Reset all types self.insn_types.fill(types::Empty); - // Fill parameter types - let entry_params = self.blocks[self.entry_block.0].params.iter(); - let param_types = self.param_types.iter(); - assert_eq!( - entry_params.len(), - entry_params.len(), - "param types should be initialized before type inference" - ); - for (param, param_type) in std::iter::zip(entry_params, param_types) { - // We know that function parameters are BasicObject or some subclass - self.insn_types[param.0] = *param_type; + // Fill entry parameter types + self.copy_param_types(); + + // Assign `new_type` to `insn` if it differs from the recorded type. + // Returns `true` if a write actually happened, `false` if the type + // Macro instead of closure so the borrow checker sees individual field + // accesses rather than an `&mut self` borrow that conflicts with + // `&self.insns` held by an outer match. + macro_rules! set_type { + ($insn:expr, $new_type:expr) => {{ + let insn = $insn; + let new_type = $new_type; + let old_type = self.insn_types[self.union_find.borrow_mut().find(insn).0]; + if old_type.bit_equal(new_type) { + false + } else { + self.insn_types[insn.0] = new_type; + true + } + }}; } - let rpo = self.rpo(); - // Walk the graph, computing types until fixpoint + let mut reachable = BlockSet::with_capacity(self.blocks.len()); - reachable.insert(self.entry_block); + reachable.insert(self.entries_block); + + // Walk the graph, computing types until fixpoint + let rpo = self.reverse_post_order(); loop { let mut changed = false; for &block in &rpo { if !reachable.get(block) { continue; } - for insn_id in &self.blocks[block.0].insns { - let insn_type = match self.find(*insn_id) { - Insn::IfTrue { val, target: BranchEdge { target, args } } => { - assert!(!self.type_of(val).bit_equal(types::Empty)); - if self.type_of(val).could_be(Type::from_cbool(true)) { - reachable.insert(target); - for (idx, arg) in args.iter().enumerate() { - let param = self.blocks[target.0].params[idx]; - self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); + for i in 0..self.blocks[block.0].insns.len() { + let insn_id = self.blocks[block.0].insns[i]; + // Instructions without output, including branch instructions, can't be targets + // of make_equal_to, so we don't need find() here. + let insn_type = match &self.insns[insn_id.0] { + Insn::CondBranch { val, if_true, if_false } => { + assert!(!self.type_of(*val).bit_equal(types::Empty)); + if self.type_of(*val).could_be(Type::from_cbool(true)) { + reachable.insert(if_true.target); + // Snapshot arg types before any param updates so phi-style + // updates happen in parallel (the args of a self-loop may name + // params of `target` itself). + let arg_types: Vec<Type> = if_true.args.iter().map(|a| self.type_of(*a)).collect(); + for (idx, arg_type) in arg_types.into_iter().enumerate() { + let param = self.blocks[if_true.target.0].params[idx]; + changed |= set_type!(param, self.type_of(param).union(arg_type)); } } - continue; - } - Insn::IfFalse { val, target: BranchEdge { target, args } } => { - assert!(!self.type_of(val).bit_equal(types::Empty)); - if self.type_of(val).could_be(Type::from_cbool(false)) { - reachable.insert(target); - for (idx, arg) in args.iter().enumerate() { - let param = self.blocks[target.0].params[idx]; - self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); + if self.type_of(*val).could_be(Type::from_cbool(false)) { + reachable.insert(if_false.target); + let arg_types: Vec<Type> = if_false.args.iter().map(|a| self.type_of(*a)).collect(); + for (idx, arg_type) in arg_types.into_iter().enumerate() { + let param = self.blocks[if_false.target.0].params[idx]; + changed |= set_type!(param, self.type_of(param).union(arg_type)); } } continue; } - Insn::Jump(BranchEdge { target, args }) => { + &Insn::Jump(BranchEdge { target, ref args }) => { reachable.insert(target); - for (idx, arg) in args.iter().enumerate() { + let arg_types: Vec<Type> = args.iter().map(|a| self.type_of(*a)).collect(); + for (idx, arg_type) in arg_types.into_iter().enumerate() { let param = self.blocks[target.0].params[idx]; - self.insn_types[param.0] = self.type_of(param).union(self.type_of(*arg)); + changed |= set_type!(param, self.type_of(param).union(arg_type)); } continue; } - insn if insn.has_output() => self.infer_type(*insn_id), + Insn::Entries { targets } => { + for &target in targets { + reachable.insert(target); + } + continue; + } + insn if insn.has_output() => self.infer_type(insn_id), _ => continue, }; - if !self.type_of(*insn_id).bit_equal(insn_type) { - self.insn_types[insn_id.0] = insn_type; - changed = true; - } + changed |= set_type!(insn_id, insn_type); } } if !changed { @@ -1467,66 +3191,308 @@ impl Function { } } - /// Return the interpreter-profiled type of the HIR instruction at the given ISEQ instruction - /// index, if it is known. This historical type record is not a guarantee and must be checked - /// with a GuardType or similar. - fn profiled_type_of_at(&self, insn: InsnId, iseq_insn_idx: usize) -> Option<ProfiledType> { - let Some(ref profiles) = self.profiles else { return None }; - let Some(entries) = profiles.types.get(&iseq_insn_idx) else { return None }; - for (entry_insn, entry_type_summary) in entries { - if self.union_find.borrow().find_const(*entry_insn) == self.union_find.borrow().find_const(insn) { - if entry_type_summary.is_monomorphic() || entry_type_summary.is_skewed_polymorphic() { - return Some(entry_type_summary.bucket(0)); + fn chase_insn(&self, insn: InsnId) -> InsnId { + let id = self.union_find.borrow().find_const(insn); + match self.insns[id.0] { + Insn::GuardType { val, .. } + | Insn::GuardBitEquals { val, .. } + | Insn::GuardAnyBitSet { val, .. } + | Insn::GuardNoBitsSet { val, .. } => self.chase_insn(val), + | Insn::RefineType { val, .. } => self.chase_insn(val), + _ => id, + } + } + + /// Return the profiled type of the HIR instruction at the given ISEQ instruction + /// index, if it is known to be monomorphic or skewed polymorphic. This historical type + /// record is not a guarantee and must be checked with a GuardType or similar. + fn profiled_type_of_at(&self, insn: InsnId, iseq_insn_idx: YarvInsnIdx) -> Option<ProfiledType> { + match self.resolve_receiver_type_from_profile(insn, iseq_insn_idx) { + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => Some(profiled_type), + _ => None, + } + } + + /// Prepare arguments for a direct send, handling keyword argument reordering and default synthesis. + /// Returns the (state, processed_args, kw_bits) to use for the SendDirect instruction, + /// or Err with the fallback reason if direct send isn't possible. + fn prepare_direct_send_args( + &mut self, + block: BlockId, + args: &[InsnId], + ci: *const rb_callinfo, + iseq: IseqPtr, + state: InsnId, + ) -> Result<(InsnId, Vec<InsnId>, u32), SendFallbackReason> { + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let (processed_args, caller_argc, kw_bits) = self.setup_keyword_arguments(block, args, kwarg, iseq)?; + + // If args were reordered or synthesized, create a new snapshot with the updated stack + let send_state = if processed_args != args { + let new_state = self.frame_state(state).with_replaced_args(&processed_args, caller_argc); + self.push_insn(block, Insn::Snapshot { state: new_state }) + } else { + state + }; + + Ok((send_state, processed_args, kw_bits)) + } + + /// Reorder keyword arguments to match the callee's expected order, and synthesize + /// default values for any optional keywords not provided by the caller. + /// + /// The output always contains all of the callee's keyword arguments (required + optional), + /// so the returned vec may be larger than the input args. + /// + /// Returns Ok with (processed_args, caller_argc, kw_bits) if successful, or Err with the fallback reason if not. + /// - caller_argc: number of arguments the caller actually pushed (for stack calculations) + /// - kw_bits: bitmask indicating which optional keywords were NOT provided by the caller + /// (used by checkkeyword to determine if non-constant defaults need evaluation) + fn setup_keyword_arguments( + &mut self, + block: BlockId, + args: &[InsnId], + kwarg: *const rb_callinfo_kwarg, + iseq: IseqPtr, + ) -> Result<(Vec<InsnId>, usize, u32), SendFallbackReason> { + let callee_keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if callee_keyword.is_null() { + if !kwarg.is_null() { + // Caller is passing kwargs but callee doesn't expect them. + return Err(SendDirectKeywordMismatch); + } + // Neither caller nor callee have keywords - nothing to do + return Ok((args.to_vec(), args.len(), 0)); + } + + // kwarg may be null if caller passes no keywords but callee has optional keywords + let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let callee_kw_count = unsafe { (*callee_keyword).num } as usize; + + // When there are 31+ keywords, CRuby uses a hash instead of a fixnum bitmask + // for kw_bits. Fall back to VM dispatch for this rare case. + if callee_kw_count >= VM_KW_SPECIFIED_BITS_MAX as usize { + return Err(SendDirectTooManyKeywords); + } + + let callee_kw_required = unsafe { (*callee_keyword).required_num } as usize; + let callee_kw_table = unsafe { (*callee_keyword).table }; + let default_values = unsafe { (*callee_keyword).default_values }; + + // Caller can't provide more keywords than callee expects (no **kwrest support yet). + if caller_kw_count > callee_kw_count { + return Err(SendDirectKeywordCountMismatch); + } + + // The keyword arguments are the last arguments in the args vector. + let kw_args_start = args.len() - caller_kw_count; + + // Build a mapping from caller keywords to their positions. + let mut caller_kw_order: Vec<ID> = Vec::with_capacity(caller_kw_count); + if !kwarg.is_null() { + for i in 0..caller_kw_count { + let sym = unsafe { get_cikw_keywords_idx(kwarg, i as i32) }; + let id = unsafe { rb_sym2id(sym) }; + caller_kw_order.push(id); + } + } + + // Verify all caller keywords are expected by callee (no unknown keywords). + // Without **kwrest, unexpected keywords should raise ArgumentError at runtime. + for &caller_id in &caller_kw_order { + let mut found = false; + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + if caller_id == expected_id { + found = true; + break; + } + } + if !found { + // Caller is passing an unknown keyword - this will raise ArgumentError. + // Fall back to VM dispatch to handle the error. + return Err(SendDirectKeywordMismatch); + } + } + + // Reorder keyword arguments to match callee expectation. + // Track which optional keywords were not provided via kw_bits. + let mut kw_bits: u32 = 0; + let mut reordered_kw_args: Vec<InsnId> = Vec::with_capacity(callee_kw_count); + for i in 0..callee_kw_count { + let expected_id = unsafe { *callee_kw_table.add(i) }; + + // Find where this keyword is in the caller's order + let mut found = false; + for (j, &caller_id) in caller_kw_order.iter().enumerate() { + if caller_id == expected_id { + reordered_kw_args.push(args[kw_args_start + j]); + found = true; + break; + } + } + + if !found { + // Required keyword not provided by caller which will raise an ArgumentError. + if i < callee_kw_required { + return Err(SendDirectMissingKeyword); + } + + // Optional keyword not provided - use default value + let default_idx = i - callee_kw_required; + let default_value = unsafe { *default_values.add(default_idx) }; + + if default_value == Qundef { + // Non-constant default (e.g., `def foo(a: compute())`). + // Set the bit so checkkeyword knows to evaluate the default at runtime. + // Push Qnil as a placeholder; the callee's checkkeyword will detect this + // and branch to evaluate the default expression. + kw_bits |= 1 << default_idx; + let nil_insn = self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + reordered_kw_args.push(nil_insn); } else { - return None; + // Constant default value - use it directly + let const_insn = self.push_insn(block, Insn::Const { val: Const::Value(default_value) }); + reordered_kw_args.push(const_insn); } } } - None + + // Replace the keyword arguments with the reordered ones. + // Keep track of the original caller argc for stack calculations. + let caller_argc = args.len(); + let mut processed_args = args[..kw_args_start].to_vec(); + processed_args.extend(reordered_kw_args); + Ok((processed_args, caller_argc, kw_bits)) } - fn likely_is_fixnum(&self, val: InsnId, profiled_type: ProfiledType) -> bool { - return self.is_a(val, types::Fixnum) || profiled_type.is_fixnum(); + /// Resolve the receiver type for method dispatch optimization. + /// + /// Takes the receiver's Type, receiver HIR instruction, and ISEQ instruction index. + /// First checks if the receiver's class is statically known, otherwise consults profile data. + /// + /// Returns: + /// - `StaticallyKnown` if the receiver's exact class is known at compile-time + /// - Result of [`Self::resolve_receiver_type_from_profile`] if we need to check profile data + fn resolve_receiver_type(&self, recv: InsnId, recv_type: Type, insn_idx: YarvInsnIdx) -> ReceiverTypeResolution { + match self.resolve_receiver_type_from_profile(recv, insn_idx) { + ReceiverTypeResolution::NoProfile => { + // Use known type information as a fallback because it doesn't have shape + // information (and we can generally eliminate duplicate guards). + if let Some(class) = recv_type.runtime_exact_ruby_class() { + ReceiverTypeResolution::StaticallyKnown { class } + } else { + ReceiverTypeResolution::NoProfile + } + } + resolution => resolution, + } } - fn coerce_to_fixnum(&mut self, block: BlockId, val: InsnId, state: InsnId) -> InsnId { - if self.is_a(val, types::Fixnum) { return val; } - return self.push_insn(block, Insn::GuardType { val, guard_type: types::Fixnum, state }); + fn polymorphic_summary(&self, profiles: &ProfileOracle, recv: InsnId, insn_idx: YarvInsnIdx) -> Option<TypeDistributionSummary> { + let Some(entries) = profiles.types.get(&insn_idx) else { + return None; + }; + let recv = self.chase_insn(recv); + for (entry_insn, entry_type_summary) in entries { + if self.union_find.borrow().find_const(*entry_insn) == recv { + if entry_type_summary.is_polymorphic() || entry_type_summary.is_skewed_polymorphic() { + return Some(entry_type_summary.clone()); + } + return None; + } + } + None } - fn arguments_likely_fixnums(&mut self, left: InsnId, right: InsnId, state: InsnId) -> bool { - let frame_state = self.frame_state(state); - let iseq_insn_idx = frame_state.insn_idx as usize; - let left_profiled_type = self.profiled_type_of_at(left, iseq_insn_idx).unwrap_or(ProfiledType::empty()); - let right_profiled_type = self.profiled_type_of_at(right, iseq_insn_idx).unwrap_or(ProfiledType::empty()); - self.likely_is_fixnum(left, left_profiled_type) && self.likely_is_fixnum(right, right_profiled_type) + /// Resolve the receiver type for method dispatch optimization from profile data. + /// + /// Returns: + /// - `Monomorphic`/`SkewedPolymorphic` if we have usable profile data + /// - `Polymorphic` if the receiver has multiple types + /// - `Megamorphic`/`SkewedMegamorphic` if the receiver has too many types to optimize + /// (SkewedMegamorphic may be optimized in the future, but for now we don't) + /// - `NoProfile` if we have no type information + fn resolve_receiver_type_from_profile(&self, recv: InsnId, insn_idx: YarvInsnIdx) -> ReceiverTypeResolution { + let Some(profiles) = self.profiles.as_ref() else { + return ReceiverTypeResolution::NoProfile; + }; + let Some(entries) = profiles.types.get(&insn_idx) else { + return ReceiverTypeResolution::NoProfile; + }; + let recv = self.chase_insn(recv); + + for (entry_insn, entry_type_summary) in entries { + if self.chase_insn(*entry_insn) == recv { + if entry_type_summary.is_monomorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::Monomorphic { profiled_type }; + } else if entry_type_summary.is_skewed_polymorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedPolymorphic { profiled_type }; + } else if entry_type_summary.is_skewed_megamorphic() { + let profiled_type = entry_type_summary.bucket(0); + return ReceiverTypeResolution::SkewedMegamorphic { profiled_type }; + } else if entry_type_summary.is_polymorphic() { + return ReceiverTypeResolution::Polymorphic; + } else if entry_type_summary.is_megamorphic() { + return ReceiverTypeResolution::Megamorphic; + } + } + } + + ReceiverTypeResolution::NoProfile } - fn try_rewrite_fixnum_op(&mut self, block: BlockId, orig_insn_id: InsnId, f: &dyn Fn(InsnId, InsnId) -> Insn, bop: u32, left: InsnId, right: InsnId, state: InsnId) { - if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, INTEGER_REDEFINED_OP_FLAG) } { - // If the basic operation is already redefined, we cannot optimize it. - self.push_insn_id(block, orig_insn_id); - return; + pub fn assume_expected_cfunc(&mut self, block: BlockId, class: VALUE, method_id: ID, cfunc: *mut c_void, state: InsnId) -> bool { + let cme = unsafe { rb_callable_method_entry(class, method_id) }; + if cme.is_null() { return false; } + let def_type = unsafe { get_cme_def_type(cme) }; + if def_type != VM_METHOD_TYPE_CFUNC { return false; } + if unsafe { get_mct_func(get_cme_def_body_cfunc(cme)) } != cfunc { + return false; } - if self.arguments_likely_fixnums(left, right, state) { - if bop == BOP_NEQ { - // For opt_neq, the interpreter checks that both neq and eq are unchanged. - self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop: BOP_EQ }, state }); - } - self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: INTEGER_REDEFINED_OP_FLAG, bop }, state }); - let left = self.coerce_to_fixnum(block, left, state); - let right = self.coerce_to_fixnum(block, right, state); - let result = self.push_insn(block, f(left, right)); - self.make_equal_to(orig_insn_id, result); - self.insn_types[result.0] = self.infer_type(result); - } else { - self.push_insn_id(block, orig_insn_id); + self.gen_patch_points_for_optimized_ccall(block, class, method_id, cme, state); + if !self.assume_no_singleton_classes(block, class, state) { + return false; } + true + } + + pub fn likely_a(&self, val: InsnId, ty: Type, state: InsnId) -> bool { + if self.type_of(val).is_subtype(ty) { + return true; + } + let frame_state = self.frame_state(state); + let iseq_insn_idx = frame_state.insn_idx; + let Some(profiled_type) = self.profiled_type_of_at(val, iseq_insn_idx) else { + return false; + }; + Type::from_profiled_type(profiled_type).is_subtype(ty) + } + + pub fn coerce_to(&mut self, block: BlockId, val: InsnId, guard_type: Type, state: InsnId) -> InsnId { + if self.is_a(val, guard_type) { return val; } + self.push_insn(block, Insn::GuardType { val, guard_type, state, recompile: None }) + } + + fn count_complex_call_features(&mut self, block: BlockId, ci_flags: c_uint) { + use Counter::*; + if 0 != ci_flags & VM_CALL_ARGS_SPLAT { self.count(block, complex_arg_pass_caller_splat); } + if 0 != ci_flags & VM_CALL_ARGS_BLOCKARG { self.count(block, complex_arg_pass_caller_blockarg); } + if 0 != ci_flags & VM_CALL_KWARG { self.count(block, complex_arg_pass_caller_kwarg); } + if 0 != ci_flags & VM_CALL_KW_SPLAT { self.count(block, complex_arg_pass_caller_kw_splat); } + if 0 != ci_flags & VM_CALL_TAILCALL { self.count(block, complex_arg_pass_caller_tailcall); } + if 0 != ci_flags & VM_CALL_SUPER { self.count(block, complex_arg_pass_caller_super); } + if 0 != ci_flags & VM_CALL_ZSUPER { self.count(block, complex_arg_pass_caller_zsuper); } + if 0 != ci_flags & VM_CALL_FORWARDING { self.count(block, complex_arg_pass_caller_forwarding); } } fn rewrite_if_frozen(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, klass: u32, bop: u32, state: InsnId) { if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { // If the basic operation is already redefined, we cannot optimize it. + self.set_dynamic_send_reason(orig_insn_id, SendWithoutBlockBopRedefined); self.push_insn_id(block, orig_insn_id); return; } @@ -1541,6 +3507,21 @@ impl Function { self.push_insn_id(block, orig_insn_id); } + pub fn try_inline_object_alloc(&mut self, block: BlockId, recv: InsnId, state: InsnId) -> Option<InsnId> { + let recv_type = self.type_of(recv); + if recv_type.is_subtype(types::Class) { + if let Some(class) = recv_type.ruby_object() { + // See class_get_alloc_func in object.c; if the class isn't initialized, is + // a singleton class, or has a custom allocator, ObjectAlloc might raise an + // exception or run arbitrary code. + if class_has_leaf_allocator(class) { + return Some(self.push_insn(block, Insn::ObjectAllocClass { class, state })); + } + } + } + None + } + fn try_rewrite_freeze(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, state: InsnId) { if self.is_a(self_val, types::StringExact) { self.rewrite_if_frozen(block, orig_insn_id, self_val, STRING_REDEFINED_OP_FLAG, BOP_FREEZE, state); @@ -1561,117 +3542,364 @@ impl Function { } } - fn try_rewrite_aref(&mut self, block: BlockId, orig_insn_id: InsnId, self_val: InsnId, idx_val: InsnId, state: InsnId) { - if !unsafe { rb_BASIC_OP_UNREDEFINED_P(BOP_AREF, ARRAY_REDEFINED_OP_FLAG) } { - // If the basic operation is already redefined, we cannot optimize it. - self.push_insn_id(block, orig_insn_id); - return; - } - let self_type = self.type_of(self_val); - let idx_type = self.type_of(idx_val); - if self_type.is_subtype(types::ArrayExact) { - if let Some(array_obj) = self_type.ruby_object() { - if array_obj.is_frozen() { - if let Some(idx) = idx_type.fixnum_value() { - self.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop: BOP_AREF }, state }); - let val = unsafe { rb_yarv_ary_entry_internal(array_obj, idx) }; - let const_insn = self.push_insn(block, Insn::Const { val: Const::Value(val) }); - self.make_equal_to(orig_insn_id, const_insn); - return; - } - } - } - } - self.push_insn_id(block, orig_insn_id); + pub fn load_rbasic_flags(&mut self, block: BlockId, recv: InsnId) -> InsnId { + // Technically this also includes the shape (_shape_id) because the (shape, flags) tuple is + // a (u32, u32) inside a u64 at RUBY_OFFSET_RBASIC_FLAGS (offset 0). It's fine to load the + // shape alongside the flags, but make sure not to *store* the shape accidentally by + // writing a u64. + self.push_insn(block, Insn::LoadField { recv, id: FieldName::RBASIC_FLAGS, offset: RUBY_OFFSET_RBASIC_FLAGS, return_type: types::CUInt64 }) + } + + fn load_ep_flags(&mut self, block: BlockId, ep: InsnId) -> InsnId { + self.push_insn(block, Insn::LoadField { recv: ep, id: FieldName::VM_ENV_DATA_INDEX_FLAGS, offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), return_type: types::CUInt64 }) + } + + pub fn guard_not_frozen(&mut self, block: BlockId, recv: InsnId, state: InsnId) { + let flags = self.load_rbasic_flags(block, recv); + self.push_insn(block, Insn::GuardNoBitsSet { val: flags, mask: Const::CUInt64(RUBY_FL_FREEZE as u64), mask_name: Some(ID!(RUBY_FL_FREEZE)), reason: SideExitReason::GuardNotFrozen, state }); + } + + pub fn guard_not_shared(&mut self, block: BlockId, recv: InsnId, state: InsnId) { + let flags = self.load_rbasic_flags(block, recv); + self.push_insn(block, Insn::GuardNoBitsSet { val: flags, mask: Const::CUInt64(RUBY_ELTS_SHARED as u64), mask_name: Some(ID!(RUBY_ELTS_SHARED)), reason: SideExitReason::GuardNotShared, state }); + } + + fn get_local_from_ep( + &mut self, + block: BlockId, + ep: InsnId, + ep_offset: u32, + level: u32, + return_type: Type, + ) -> InsnId { + let local_id = get_local_var_id(self.iseq, level, ep_offset); + let ep_offset = i32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to i32")); + let offset = -(SIZEOF_VALUE_I32 * ep_offset); + + self.push_insn(block, Insn::LoadField { + recv: ep, + id: local_id.into(), + offset, + return_type, + }) + } + + fn get_local_from_sp( + &mut self, + block: BlockId, + sp: InsnId, + ep_offset: u32, + return_type: Type, + ) -> InsnId { + let local_id = get_local_var_id(self.iseq, 0, ep_offset); + let ep_offset = i32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to i32")); + let offset = -(SIZEOF_VALUE_I32 * (ep_offset + 1)); + + self.push_insn(block, Insn::LoadField { + recv: sp, + id: local_id.into(), + offset, + return_type, + }) } - /// Rewrite SendWithoutBlock opcodes into SendWithoutBlockDirect opcodes if we know the target - /// ISEQ statically. This removes run-time method lookups and opens the door for inlining. - fn optimize_direct_sends(&mut self) { - for block in self.rpo() { + /// Rewrite eligible Send opcodes into SendDirect + /// opcodes if we know the target ISEQ statically. This removes run-time method lookups and + /// opens the door for inlining. + /// Also try and inline constant caches, specialize object allocations, and more. + /// Calls to C functions are handled separately in optimize_c_calls. + fn type_specialize(&mut self) { + for block in self.reverse_post_order() { let old_insns = std::mem::take(&mut self.blocks[block.0].insns); assert!(self.blocks[block.0].insns.is_empty()); for insn_id in old_insns { match self.find(insn_id) { - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(plus) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumAdd { left, right, state }, BOP_PLUS, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minus) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumSub { left, right, state }, BOP_MINUS, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(mult) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumMult { left, right, state }, BOP_MULT, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(div) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumDiv { left, right, state }, BOP_DIV, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(modulo) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumMod { left, right, state }, BOP_MOD, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(eq) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumEq { left, right }, BOP_EQ, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(neq) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumNeq { left, right }, BOP_NEQ, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(lt) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumLt { left, right }, BOP_LT, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(le) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumLe { left, right }, BOP_LE, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(gt) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumGt { left, right }, BOP_GT, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(ge) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumGe { left, right }, BOP_GE, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(and) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumAnd { left, right }, BOP_AND, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(or) && args.len() == 1 => - self.try_rewrite_fixnum_op(block, insn_id, &|left, right| Insn::FixnumOr { left, right }, BOP_OR, self_val, args[0], state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(freeze) && args.len() == 0 => - self.try_rewrite_freeze(block, insn_id, self_val, state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.len() == 0 => - self.try_rewrite_uminus(block, insn_id, self_val, state), - Insn::SendWithoutBlock { self_val, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(aref) && args.len() == 1 => - self.try_rewrite_aref(block, insn_id, self_val, args[0], state), - Insn::SendWithoutBlock { mut self_val, cd, args, state } => { + Insn::Send { recv, block: None, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(freeze) && args.is_empty() => + self.try_rewrite_freeze(block, insn_id, recv, state), + Insn::Send { recv, block: None, args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => + self.try_rewrite_uminus(block, insn_id, recv, state), + Insn::Send { mut recv, cd, state, block: send_block, args, .. } => { + let has_block = send_block.is_some(); let frame_state = self.frame_state(state); - let (klass, profiled_type) = if let Some(klass) = self.type_of(self_val).runtime_exact_ruby_class() { - // If we know the class statically, use it to fold the lookup at compile-time. - (klass, None) - } else { - // If we know that self is reasonably monomorphic from profile information, guard and use it to fold the lookup at compile-time. - // TODO(max): Figure out how to handle top self? - let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { - self.push_insn_id(block, insn_id); continue; - }; - (recv_type.class(), Some(recv_type)) + let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), frame_state.insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } + | ReceiverTypeResolution::Megamorphic => { + if get_option!(stats) { + let reason = if has_block { SendMegamorphic } else { SendWithoutBlockMegamorphic }; + self.set_dynamic_send_reason(insn_id, reason); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::Polymorphic => { + if get_option!(stats) { + let reason = if has_block { SendPolymorphic } else { SendWithoutBlockPolymorphic }; + self.set_dynamic_send_reason(insn_id, reason); + } + self.push_insn_id(block, insn_id); + continue; + } + ReceiverTypeResolution::NoProfile => { + let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); + continue; + } }; let ci = unsafe { get_call_data_ci(cd) }; // info about the call site + + let flags = unsafe { rb_vm_ci_flag(ci) }; + let mid = unsafe { vm_ci_mid(ci) }; // Do method lookup let mut cme = unsafe { rb_callable_method_entry(klass, mid) }; if cme.is_null() { + let reason = if has_block { SendNotOptimizedMethodType(MethodType::Null) } else { SendWithoutBlockNotOptimizedMethodType(MethodType::Null) }; + self.set_dynamic_send_reason(insn_id, reason); self.push_insn_id(block, insn_id); continue; } // Load an overloaded cme if applicable. See vm_search_cc(). // It allows you to use a faster ISEQ if possible. cme = unsafe { rb_check_overloaded_cme(cme, ci) }; - let def_type = unsafe { get_cme_def_type(cme) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + let reason = if has_block { SendNotOptimizedNeedPermission } else { SendWithoutBlockNotOptimizedNeedPermission }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + } + } + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } + + // If the call site info indicates that the `Function` has overly complex arguments, then do not optimize into a `SendDirect`. + // Optimized methods(`VM_METHOD_TYPE_OPTIMIZED`) handle their own argument constraints (e.g., kw_splat for Proc call). + if def_type != VM_METHOD_TYPE_OPTIMIZED && unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + if def_type == VM_METHOD_TYPE_ISEQ { // TODO(max): Allow non-iseq; cache cme // Only specialize positional-positional calls // TODO(max): Handle other kinds of parameter passing let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; - if !can_direct_send(iseq) { + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice(), has_block) { + self.push_insn_id(block, insn_id); continue; + } + + // Check if the args are compatible before emitting any assmptions + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); self.push_insn_id(block, insn_id); continue; } + + // Add PatchPoint for method redefinition self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + + // Add GuardType for profiled receiver if let Some(profiled_type) = profiled_type { - self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: Type::from_profiled_type(profiled_type), state }); + let argc = unsafe { vm_ci_argc(ci) } as i32; + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend { argc }) }); } - let send_direct = self.push_insn(block, Insn::SendWithoutBlockDirect { self_val, cd, cme, iseq, args, state }); + + let send_direct = self.push_insn(block, Insn::SendDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state, block: send_block }); self.make_equal_to(insn_id, send_direct); - } else if def_type == VM_METHOD_TYPE_IVAR && args.is_empty() { + } else if !has_block && def_type == VM_METHOD_TYPE_BMETHOD { + let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; + let proc = unsafe { rb_jit_get_proc_ptr(procv) }; + let proc_block = unsafe { &(*proc).block }; + // Target ISEQ bmethods. Can't handle for example, `define_method(:foo, &:foo)` + // which makes a `block_type_symbol` bmethod. + if proc_block.type_ != block_type_iseq { + self.set_dynamic_send_reason(insn_id, BmethodNonIseqProc); + self.push_insn_id(block, insn_id); continue; + } + let capture = unsafe { proc_block.as_.captured.as_ref() }; + let iseq = unsafe { *capture.code.iseq.as_ref() }; + + if !can_direct_send(self, block, iseq, ci, insn_id, args.as_slice(), has_block) { + self.push_insn_id(block, insn_id); continue; + } + + // Check if the args are compatible before emitting any assmptions + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + // Patch points: + // Check for "defined with an un-shareable Proc in a different Ractor" + if !procv.shareable_p() && !self.assume_single_ractor_mode(block, state) { + // TODO(alan): Turn this into a ractor belonging guard to work better in multi ractor mode. + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { - self_val = self.push_insn(block, Insn::GuardType { val: self_val, guard_type: Type::from_profiled_type(profiled_type), state }); + let argc = unsafe { vm_ci_argc(ci) } as i32; + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend{ argc }) }); + } + + let send_direct = self.push_insn(block, Insn::SendDirect { recv, cd, cme, iseq, args: processed_args, kw_bits, state: send_state, block: None }); + self.make_equal_to(insn_id, send_direct); + } else if !has_block && def_type == VM_METHOD_TYPE_IVAR && args.is_empty() { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + let argc = unsafe { vm_ci_argc(ci) } as i32; + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend{ argc }) }); } let id = unsafe { get_cme_def_body_attr_id(cme) }; - let getivar = self.push_insn(block, Insn::GetIvar { self_val, id, state }); + + let getivar = self.push_insn(block, Insn::GetIvar { self_val: recv, id, ic: std::ptr::null(), state }); self.make_equal_to(insn_id, getivar); + } else if let (false, VM_METHOD_TYPE_ATTRSET, &[val]) = (has_block, def_type, args.as_slice()) { + // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. + // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. + if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); + self.push_insn_id(block, insn_id); continue; + } + + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + let argc = unsafe { vm_ci_argc(ci) } as i32; + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend{ argc }) }); + } + let id = unsafe { get_cme_def_body_attr_id(cme) }; + + self.push_insn(block, Insn::SetIvar { self_val: recv, id, ic: std::ptr::null(), val, state }); + self.make_equal_to(insn_id, val); + } else if !has_block && def_type == VM_METHOD_TYPE_OPTIMIZED { + let opt_type: OptimizedMethodType = unsafe { get_cme_def_body_optimized_type(cme) }.into(); + match (opt_type, args.as_slice()) { + (OptimizedMethodType::Call, _) => { + if flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KWARG) != 0 { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + let argc = unsafe { vm_ci_argc(ci) } as i32; + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend{ argc }) }); + } + let kw_splat = flags & VM_CALL_KW_SPLAT != 0; + let invoke_proc = self.push_insn(block, Insn::InvokeProc { recv, args: args.clone(), state, kw_splat }); + self.make_equal_to(insn_id, invoke_proc); + } + (OptimizedMethodType::StructAref, &[]) | (OptimizedMethodType::StructAset, &[_]) => { + if unspecializable_call_type(flags) { + self.count_complex_call_features(block, flags); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } + let index: i32 = unsafe { get_cme_def_body_optimized_index(cme) } + .try_into() + .unwrap(); + // We are going to use an encoding that takes a 4-byte immediate which + // limits the offset to INT32_MAX. + { + let native_index = (index as i64) * (SIZEOF_VALUE as i64); + if native_index > (i32::MAX as i64) { + self.set_dynamic_send_reason(insn_id, TooManyArgsForLir); + self.push_insn_id(block, insn_id); continue; + } + } + // Get the profiled type to check if the fields is embedded or heap allocated. + let Some(is_embedded) = self.profiled_type_of_at(recv, frame_state.insn_idx).map(|t| t.flags().is_struct_embedded()) else { + // No (monomorphic/skewed polymorphic) profile info + let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles }; + self.set_dynamic_send_reason(insn_id, reason); + self.push_insn_id(block, insn_id); continue; + }; + // Check singleton class assumption first, before emitting other patchpoints + if !self.assume_no_singleton_classes(block, klass, state) { + self.set_dynamic_send_reason(insn_id, SingletonClassSeen); + self.push_insn_id(block, insn_id); continue; + } + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(profiled_type) = profiled_type { + let argc = unsafe { vm_ci_argc(ci) } as i32; + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend{ argc }) }); + } + // All structs from the same Struct class should have the same + // length. So if our recv is embedded all runtime + // structs of the same class should be as well, and the same is + // true of the converse. + // + // No need for a GuardShape. + if let OptimizedMethodType::StructAset = opt_type { + self.guard_not_frozen(block, recv, state); + } + + let (target, offset) = if is_embedded { + let offset = RUBY_OFFSET_RSTRUCT_AS_ARY + (SIZEOF_VALUE_I32 * index); + (recv, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv, id: FieldName::as_heap, offset: RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * index; + (as_heap, offset) + }; + + let replacement = if let (OptimizedMethodType::StructAset, &[val]) = (opt_type, args.as_slice()) { + self.push_insn(block, Insn::StoreField { recv: target, id: mid.into(), offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv, val }); + val + } else { // StructAref + self.push_insn(block, Insn::LoadField { recv: target, id: mid.into(), offset, return_type: types::BasicObject }) + }; + self.make_equal_to(insn_id, replacement); + }, + _ => { + self.set_dynamic_send_reason(insn_id, SendWithoutBlockNotOptimizedMethodTypeOptimized(OptimizedMethodType::from(opt_type))); + self.push_insn_id(block, insn_id); continue; + }, + }; } else { + let reason = if has_block { SendNotOptimizedMethodType(MethodType::from(def_type)) } else { SendWithoutBlockNotOptimizedMethodType(MethodType::from(def_type)) }; + self.set_dynamic_send_reason(insn_id, reason); self.push_insn_id(block, insn_id); continue; } } @@ -1682,12 +3910,9 @@ impl Function { self.push_insn_id(block, insn_id); continue; } let cref_sensitive = !unsafe { (*ice).ic_cref }.is_null(); - let multi_ractor_mode = unsafe { rb_zjit_multi_ractor_p() }; - if cref_sensitive || multi_ractor_mode { + if cref_sensitive || !self.assume_single_ractor_mode(block, state) { self.push_insn_id(block, insn_id); continue; } - // Assume single-ractor mode. - self.push_insn(block, Insn::PatchPoint { invariant: Invariant::SingleRactorMode, state }); // Invalidate output code on any constant writes associated with constants // referenced after the PatchPoint. self.push_insn(block, Insn::PatchPoint { invariant: Invariant::StableConstantNames { idlist }, state }); @@ -1695,12 +3920,27 @@ impl Function { self.insn_types[replacement.0] = self.infer_type(replacement); self.make_equal_to(insn_id, replacement); } - Insn::ObjToString { val, .. } => { + Insn::ObjToString { val, cd, state, .. } => { if self.is_a(val, types::String) { - // behaves differently from `SendWithoutBlock` with `mid:to_s` because ObjToString should not have a patch point for String to_s being redefined - self.make_equal_to(insn_id, val); + // behaves differently from `Send` with `mid:to_s` because ObjToString should not have a patch point for String to_s being redefined + self.make_equal_to(insn_id, val); continue; + } + + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(val, frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); continue + }; + + if recv_type.is_string() { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoSingletonClass { klass: recv_type.class() }, state }); + let guard = self.push_insn(block, Insn::GuardType { val, guard_type: types::String, state, recompile: None }); + // Infer type so AnyToString can fold off this + self.insn_types[guard.0] = self.infer_type(guard); + self.make_equal_to(insn_id, guard); } else { - self.push_insn_id(block, insn_id); + let recv = self.push_insn(block, Insn::GuardType { val, guard_type: Type::from_profiled_type(recv_type), state, recompile: None }); + let send_to_s = self.push_insn(block, Insn::Send { recv, cd, block: None, args: vec![], state, reason: ObjToStringNotString }); + self.make_equal_to(insn_id, send_to_s); } } Insn::AnyToString { str, .. } => { @@ -1710,25 +3950,736 @@ impl Function { self.push_insn_id(block, insn_id); } } + Insn::IsMethodCfunc { val, cd, cfunc, state } if self.type_of(val).ruby_object_known() => { + let class = self.type_of(val).ruby_object().unwrap(); + let cme = unsafe { rb_zjit_vm_search_method(self.iseq.into(), cd as *mut rb_call_data, class) }; + let is_expected_cfunc = unsafe { rb_zjit_cme_is_cfunc(cme, cfunc as *const c_void) }; + let method = unsafe { rb_vm_ci_mid((*cd).ci) }; + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: class, method, cme }, state }); + let replacement = self.push_insn(block, Insn::Const { val: Const::CBool(is_expected_cfunc) }); + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } + Insn::ObjectAlloc { val, state } => { + if let Some(replacement) = self.try_inline_object_alloc(block, val, state) { + self.insn_types[replacement.0] = self.infer_type(replacement); + self.make_equal_to(insn_id, replacement); + } else { + self.push_insn_id(block, insn_id); + } + } + Insn::NewRange { low, high, flag, state } => { + let low_is_fix = self.is_a(low, types::Fixnum); + let high_is_fix = self.is_a(high, types::Fixnum); + + if low_is_fix || high_is_fix { + let low_fix = self.coerce_to(block, low, types::Fixnum, state); + let high_fix = self.coerce_to(block, high, types::Fixnum, state); + let replacement = self.push_insn(block, Insn::NewRangeFixnum { low: low_fix, high: high_fix, flag, state }); + self.make_equal_to(insn_id, replacement); + self.insn_types[replacement.0] = self.infer_type(replacement); + } else { + self.push_insn_id(block, insn_id); + }; + } + Insn::InvokeSuper { recv, cd, blockiseq, args, state, .. } => { + // Helper to emit common guards for super call optimization. + fn emit_super_call_guards( + fun: &mut Function, + block: BlockId, + super_cme: *const rb_callable_method_entry_t, + current_cme: *const rb_callable_method_entry_t, + mid: ID, + state: InsnId, + ) { + fun.push_insn(block, Insn::PatchPoint { + invariant: Invariant::MethodRedefined { + klass: unsafe { (*super_cme).defined_class }, + method: mid, + cme: super_cme + }, + state + }); + + // Get the EP of the ISeq of the containing method, or "local level", skipping over block-level EPs. + // Equivalent of GET_LEP() macro. + let level = get_lvar_level(fun.iseq); + let lep = fun.push_insn(block, Insn::GetEP { level }); + // Load ep[VM_ENV_DATA_INDEX_ME_CREF] + let method_entry = fun.push_insn(block, Insn::LoadField { recv: lep, id: FieldName::VM_ENV_DATA_INDEX_ME_CREF, offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_ME_CREF, return_type: types::RubyValue }); + // Guard that it matches the expected CME + fun.push_insn(block, Insn::GuardBitEquals { val: method_entry, expected: Const::Value(current_cme.into()), reason: SideExitReason::GuardSuperMethodEntry, state, recompile: None }); + + let block_handler = fun.push_insn(block, Insn::LoadField { recv: lep, id: FieldName::VM_ENV_DATA_INDEX_SPECVAL, offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, return_type: types::RubyValue }); + fun.push_insn(block, Insn::GuardBitEquals { + val: block_handler, + expected: Const::Value(VALUE(VM_BLOCK_HANDLER_NONE as usize)), + reason: SideExitReason::UnhandledBlockArg, + state, + recompile: None, + }); + } + + // Don't handle calls with literal blocks (e.g., super { ... }) + if !blockiseq.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperCallWithBlock); + continue; + } + + let frame_state = self.frame_state(state); + + // Don't handle super in a block since that needs a loop to find the running CME. + if frame_state.iseq != unsafe { rb_get_iseq_body_local_iseq(frame_state.iseq) } { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperFromBlock); + continue; + } + + let ci = unsafe { get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + assert!(flags & VM_CALL_FCALL != 0); + + // Reject calls with complex argument handling. + if unspecializable_c_call_type(flags) { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperComplexArgsPass); + continue; + } + + // Get the profiled CME from the current method. + let Some(profiles) = self.profiles.as_ref() else { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNoProfiles); + continue; + }; + + let Some(current_cme) = profiles.payload.profile.get_super_method_entry(frame_state.insn_idx) else { + self.push_insn_id(block, insn_id); + + // The absence of the super CME could be due to a missing profile, but + // if we've made it this far the value would have been deleted, indicating + // that the call is at least polymorphic and possibly megamorphic. + self.set_dynamic_send_reason(insn_id, SuperPolymorphic); + continue; + }; + + // Get defined_class and method ID from the profiled CME. + let current_defined_class = unsafe { (*current_cme).defined_class }; + let mid = unsafe { get_def_original_id((*current_cme).def) }; + + // Compute superclass: RCLASS_SUPER(RCLASS_ORIGIN(defined_class)) + let superclass = unsafe { rb_class_get_superclass(RCLASS_ORIGIN(current_defined_class)) }; + if superclass.nil_p() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperClassNotFound); + continue; + } + + // Look up the super method. + let mut super_cme = unsafe { rb_callable_method_entry(superclass, mid) }; + if super_cme.is_null() { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetNotFound); + continue; + } + + let mut def_type = unsafe { get_cme_def_type(super_cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + super_cme = unsafe { rb_aliased_callable_method_entry(super_cme) }; + def_type = unsafe { get_cme_def_type(super_cme) }; + } + + if def_type == VM_METHOD_TYPE_ISEQ { + // Check if the super method's parameters support direct send. + // If not, we can't do direct dispatch. + let super_iseq = unsafe { get_def_iseq_ptr((*super_cme).def) }; + // TODO: pass Option<blockiseq> to can_direct_send when we start specializing `super { ... }`. + if !can_direct_send(self, block, super_iseq, ci, insn_id, args.as_slice(), false) { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperTargetComplexArgsPass); + continue; + } + + // Check if the args are compatible before emitting any assmptions + let Ok((send_state, processed_args, kw_bits)) = self.prepare_direct_send_args(block, &args, ci, super_iseq, state) + .inspect_err(|&reason| self.set_dynamic_send_reason(insn_id, reason)) else { + self.push_insn_id(block, insn_id); continue; + }; + + emit_super_call_guards(self, block, super_cme, current_cme, mid, state); + + // Use SendDirect with the super method's CME and ISEQ. + let send_direct = self.push_insn(block, Insn::SendDirect { + recv, + cd, + cme: super_cme, + iseq: super_iseq, + args: processed_args, + kw_bits, + state: send_state, + block: None, + }); + self.make_equal_to(insn_id, send_direct); + + } else if def_type == VM_METHOD_TYPE_CFUNC { + let cfunc = unsafe { get_cme_def_body_cfunc(super_cme) }; + let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast(); + + let props = ZJITState::get_method_annotations().get_cfunc_properties(super_cme); + if props.is_none() && get_option!(stats) { + self.count_not_annotated_cfunc(block, super_cme); + } + let props = props.unwrap_or_default(); + + match cfunc_argc { + // C function with fixed argument count. + 0.. => { + // Check argc matches + if args.len() != cfunc_argc as usize { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, ArgcParamMismatch); + continue; + } + + emit_super_call_guards(self, block, super_cme, current_cme, mid, state); + + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } + + // Use CCallWithFrame for the C function. + let name = unsafe { (*super_cme).called_id }; + let owner = unsafe { (*super_cme).owner }; + let return_type = props.return_type; + let elidable = props.elidable; + // Filter for a leaf and GC free function + let ccall = if props.leaf && props.no_gc { + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }) + } else { + if get_option!(stats) { + self.count_not_inlined_cfunc(block, super_cme); + } + self.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc: cfunc_ptr, + recv, + args: args.clone(), + cme: super_cme, + name, + state, + return_type: types::BasicObject, + elidable: false, + block: None, + }) + }; + self.make_equal_to(insn_id, ccall); + } + + // Variadic C function: func(int argc, VALUE *argv, VALUE recv) + -1 => { + emit_super_call_guards(self, block, super_cme, current_cme, mid, state); + + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } + + // Use CCallVariadic for the variadic C function. + let name = unsafe { (*super_cme).called_id }; + let owner = unsafe { (*super_cme).owner }; + let return_type = props.return_type; + let elidable = props.elidable; + // Filter for a leaf and GC free function + let ccall = if props.leaf && props.no_gc { + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }) + } else { + if get_option!(stats) { + self.count_not_inlined_cfunc(block, super_cme); + } + self.push_insn(block, Insn::CCallVariadic { + cfunc: cfunc_ptr, + recv, + args: args.clone(), + cme: super_cme, + name, + state, + return_type: types::BasicObject, + elidable: false, + block: None, + }) + }; + self.make_equal_to(insn_id, ccall); + } + + // Array-variadic: (self, args_ruby_array). + -2 => { + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNotOptimizedMethodType(MethodType::Cfunc)); + continue; + } + _ => unreachable!("unknown cfunc argc: {}", cfunc_argc) + } + } else { + // Other method types (not ISEQ or CFUNC) + self.push_insn_id(block, insn_id); + self.set_dynamic_send_reason(insn_id, SuperNotOptimizedMethodType(MethodType::from(def_type))); + continue; + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + fn inline(&mut self) { + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + // We can inline SendDirect with blockiseq because we are prohibiting `yield` + // and `.call`, which would trigger autosplat. We only inline constants and + // variables and builtin calls. + Insn::SendDirect { recv, iseq, cd, args, state, .. } => { + let call_info = unsafe { (*cd).ci }; + let ci_flags = unsafe { vm_ci_flag(call_info) }; + // .send call is not currently supported for builtins + if ci_flags & VM_CALL_OPT_SEND != 0 { + self.push_insn_id(block, insn_id); continue; + } + let Some(value) = iseq_get_return_value(iseq, None, ci_flags) else { + self.push_insn_id(block, insn_id); continue; + }; + match value { + IseqReturn::LocalVariable(idx) => { + self.count(block, Counter::inline_iseq_optimized_send_count); + self.make_equal_to(insn_id, args[idx as usize]); + } + IseqReturn::Value(value) => { + self.count(block, Counter::inline_iseq_optimized_send_count); + let replacement = self.push_insn(block, Insn::Const { val: Const::Value(value) }); + self.make_equal_to(insn_id, replacement); + } + IseqReturn::Receiver => { + self.count(block, Counter::inline_iseq_optimized_send_count); + self.make_equal_to(insn_id, recv); + } + IseqReturn::InvokeLeafBuiltin(bf, return_type) => { + self.count(block, Counter::inline_iseq_optimized_send_count); + let replacement = self.push_insn(block, Insn::InvokeBuiltin { + bf, + recv, + args: vec![recv], + state, + leaf: true, + return_type, + }); + self.make_equal_to(insn_id, replacement); + } + } + } + _ => { self.push_insn_id(block, insn_id); } + } + } + } + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + fn load_shape(&mut self, block: BlockId, recv: InsnId) -> InsnId { + self.push_insn(block, Insn::LoadField { + recv, + id: FieldName::shape_id, + offset: unsafe { rb_shape_id_offset() } as i32, + return_type: types::CShape + }) + } + + fn guard_shape(&mut self, block: BlockId, val: InsnId, expected: ShapeId, state: InsnId, recompile: Option<Recompile>) -> InsnId { + self.push_insn(block, Insn::GuardBitEquals { + val, + expected: Const::CShape(expected), + reason: SideExitReason::GuardShape(expected), + state, + recompile, + }) + } + + fn load_ivar_c_call(&mut self, block: BlockId, recv: InsnId, ivar_index: attr_index_t) -> InsnId { + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + let ivar_index_insn = self.push_insn(block, Insn::Const { val: Const::CAttrIndex(ivar_index) }); + self.push_insn(block, Insn::CCall { + cfunc: rb_ivar_get_at_no_ractor_check as *const u8, + recv, + args: vec![ivar_index_insn], + name: ID!(rb_ivar_get_at_no_ractor_check), + owner: Qnil, + return_type: types::BasicObject, + elidable: true }) + } + + fn load_ivar_heap(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: attr_index_t) -> InsnId { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let ptr = self.push_insn(block, Insn::LoadField { + recv, id: FieldName::as_heap, + offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, + return_type: types::CPtr, + }); + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + self.push_insn(block, Insn::LoadField { + recv: ptr, id: id.into(), offset, + return_type: types::BasicObject, + }) + } + + fn load_ivar_embedded(&mut self, block: BlockId, recv: InsnId, id: ID, ivar_index: attr_index_t) -> InsnId { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + self.push_insn(block, Insn::LoadField { + recv, id: id.into(), offset, + return_type: types::BasicObject, + }) + } + + fn load_ivar_from_fields(&mut self, block: BlockId, recv: InsnId, is_embedded: bool, id: ID, ivar_index: attr_index_t) -> InsnId { + if is_embedded { + return self.load_ivar_embedded(block, recv, id, ivar_index); + } else { + return self.load_ivar_heap(block, recv, id, ivar_index); + } + } + + /// This puts a guard that establishes the preconditon for [Self::load_ivar] + fn load_ivar_guard_type(&mut self, block: BlockId, recv: InsnId, recv_type: ProfiledType, state: InsnId) -> InsnId { + if recv_type.flags().is_t_class() { + // Check class first since `Class < Module` + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::Class, state, recompile: None }) + } else if recv_type.flags().is_t_module() { + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::Module, state, recompile: None }) + } else if recv_type.flags().is_t_data() { + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::TData, state, recompile: None }) + } else { + // HeapBasicObject is wider than T_OBJECT, but shapes for T_OBJECTs are in a pool of + // its own and are guaranteed to be different from shapes of any other T_* types. So + // the shape check that follows already covers checking for T_OBJECT. + self.push_insn(block, Insn::GuardType { val: recv, guard_type: types::HeapBasicObject, state, recompile: None }) + } + } + + fn load_ivar(&mut self, block: BlockId, self_val: InsnId, recv_type: ProfiledType, id: ID, state: InsnId) -> InsnId { + // Too-complex shapes use hash tables; rb_shape_get_iv_index doesn't support them. + // Callers must filter these out before calling load_ivar. + assert!(!recv_type.shape().is_complex(), "load_ivar called with too-complex shape"); + let mut ivar_index: attr_index_t = 0; + if ! unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + return self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + } + if recv_type.flags().is_t_class() || recv_type.flags().is_t_module() { + // Class/module ivar: load from prime classext's fields_obj + if !self.assume_root_box(block, state) { + // Non-root box active: fall back to C call + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + return self.load_ivar_c_call(block, self_val, ivar_index); + } + // Root box only: load directly from prime classext + let fields_obj = self.push_insn(block, Insn::LoadField { + recv: self_val, id: FieldName::fields_obj, + offset: RCLASS_OFFSET_PRIME_FIELDS_OBJ as i32, + return_type: types::RubyValue, + }); + return self.load_ivar_from_fields(block, fields_obj, recv_type.flags().is_fields_embedded(), id, ivar_index); + } + if recv_type.flags().is_t_data() { + let fields_obj = self.push_insn(block, Insn::LoadField { + recv: self_val, id: FieldName::fields_obj, + offset: TDATA_OFFSET_FIELDS_OBJ as i32, + return_type: types::RubyValue, + }); + return self.load_ivar_from_fields(block, fields_obj, recv_type.flags().is_fields_embedded(), id, ivar_index); + } + if recv_type.flags().is_t_object() { + return self.load_ivar_from_fields(block, self_val, recv_type.flags().is_embedded(), id, ivar_index); + } + // Non-T_OBJECT, non-class/module, non-typed-data: fall back to C call + // NOTE: it's fine to use rb_ivar_get_at_no_ractor_check because + // getinstancevariable does assume_single_ractor_mode() + return self.load_ivar_c_call(block, self_val, ivar_index); + } + + fn optimize_getivar(&mut self) { + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::GetIvar { self_val, id, ic: _, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.count(block, Counter::getivar_fallback_not_monomorphic); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.count(block, Counter::getivar_fallback_immediate); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if recv_type.shape().is_complex() { + // too-complex shapes can't use index access + self.count(block, Counter::getivar_fallback_complex); + self.push_insn_id(block, insn_id); continue; + } + if self.policy.no_side_exits { + // On the final version, skip GetIvar shape specialization. + // iseq_to_hir already generates polymorphic branches with a + // GetIvar C call fallback for getinstancevariable, so we don't + // need to wrap it again here. + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.load_ivar_guard_type(block, self_val, recv_type, state); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state, Some(Recompile::ProfileSelf)); + let replacement = self.load_ivar(block, self_val, recv_type, id, state); + self.make_equal_to(insn_id, replacement); + } + Insn::DefinedIvar { self_val, id, pushval, state } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.count(block, Counter::definedivar_fallback_not_monomorphic); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.count(block, Counter::definedivar_fallback_immediate); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.count(block, Counter::definedivar_fallback_not_t_object); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_complex() { + // too-complex shapes can't use index access + self.count(block, Counter::definedivar_fallback_complex); + self.push_insn_id(block, insn_id); continue; + } + if self.policy.no_side_exits { + // On the final version, keep the DefinedIvar fallback instead of another shape guard. + self.push_insn_id(block, insn_id); continue; + } + let self_val = self.load_ivar_guard_type(block, self_val, recv_type, state); + let shape = self.load_shape(block, self_val); + self.guard_shape(block, shape, recv_type.shape(), state, Some(Recompile::ProfileSelf)); + let mut ivar_index: attr_index_t = 0; + let replacement = if unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + self.push_insn(block, Insn::Const { val: Const::Value(pushval) }) + } else { + // If there is no IVAR index, then the ivar was undefined when we + // entered the compiler. That means we can just return nil for this + // shape + iv name + self.push_insn(block, Insn::Const { val: Const::Value(Qnil) }) + }; + self.make_equal_to(insn_id, replacement); + } + Insn::SetIvar { self_val, id, val, state, ic } => { + let frame_state = self.frame_state(state); + let Some(recv_type) = self.profiled_type_of_at(self_val, frame_state.insn_idx) else { + // No (monomorphic/skewed polymorphic) profile info + self.count(block, Counter::setivar_fallback_not_monomorphic); + self.push_insn_id(block, insn_id); continue; + }; + if recv_type.flags().is_immediate() { + // Instance variable lookups on immediate values are always nil + self.count(block, Counter::setivar_fallback_immediate); + self.push_insn_id(block, insn_id); continue; + } + assert!(recv_type.shape().is_valid()); + if !recv_type.flags().is_t_object() { + // Check if the receiver is a T_OBJECT + self.count(block, Counter::setivar_fallback_not_t_object); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_complex() { + // too-complex shapes can't use index access + self.count(block, Counter::setivar_fallback_complex); + self.push_insn_id(block, insn_id); continue; + } + if recv_type.shape().is_frozen() { + // Can't set ivars on frozen objects + self.count(block, Counter::setivar_fallback_frozen); + self.push_insn_id(block, insn_id); continue; + } + if self.policy.no_side_exits { + // TODO: Support polymorphic SetIvar shape-specialized paths. + // https://github.com/Shopify/ruby/issues/927 + // On the final version, keep the SetIvar fallback instead of another shape guard. + self.push_insn_id(block, insn_id); continue; + } + let mut ivar_index: attr_index_t = 0; + let mut next_shape_id = recv_type.shape(); + if !unsafe { rb_shape_get_iv_index(recv_type.shape().0, id, &mut ivar_index) } { + // Current shape does not contain this ivar; do a shape transition. + let current_shape_id = recv_type.shape(); + let class = recv_type.class(); + // We're only looking at T_OBJECT so ignore all of the imemo stuff. + assert!(recv_type.flags().is_t_object()); + next_shape_id = ShapeId(unsafe { rb_shape_transition_add_ivar_no_warnings(current_shape_id.0, id, class) }); + // If the VM ran out of shapes, or this class generated too many leaf, + // it may be de-optimized into OBJ_COMPLEX_SHAPE (hash-table). + let new_shape_complex = unsafe { rb_jit_shape_complex_p(next_shape_id.0) }; + // TODO(max): Is it OK to bail out here after making a shape transition? + if new_shape_complex { + self.count(block, Counter::setivar_fallback_new_shape_complex); + self.push_insn_id(block, insn_id); continue; + } + let ivar_result = unsafe { rb_shape_get_iv_index(next_shape_id.0, id, &mut ivar_index) }; + assert!(ivar_result, "New shape must have the ivar index"); + let current_capacity = unsafe { rb_jit_shape_capacity(current_shape_id.0) }; + let next_capacity = unsafe { rb_jit_shape_capacity(next_shape_id.0) }; + // If the new shape has a different capacity, or is COMPLEX, we'll have to + // reallocate it. + let needs_extension = next_capacity != current_capacity; + if needs_extension { + self.count(block, Counter::setivar_fallback_new_shape_needs_extension); + self.push_insn_id(block, insn_id); continue; + } + // Fall through to emitting the ivar write + } + let self_val = self.load_ivar_guard_type(block, self_val, recv_type, state); + let shape = self.load_shape(block, self_val); + // TODO: attr_writer SetIvar has a null inline cache and may target a receiver + // operand other than CFP self. Support it with a reprofile strategy that + // profiles the receiver operand even after the send insn has finished profiling. + let recompile = if ic.is_null() { None } else { Some(Recompile::ProfileSelf) }; + self.guard_shape(block, shape, recv_type.shape(), state, recompile); + // Current shape contains this ivar + let (ivar_storage, offset) = if recv_type.flags().is_embedded() { + // See ROBJECT_FIELDS() from include/ruby/internal/core/robject.h + let offset = ROBJECT_OFFSET_AS_ARY as i32 + (SIZEOF_VALUE * ivar_index.to_usize()) as i32; + (self_val, offset) + } else { + let as_heap = self.push_insn(block, Insn::LoadField { recv: self_val, id: FieldName::as_heap, offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, return_type: types::CPtr }); + let offset = SIZEOF_VALUE_I32 * ivar_index as i32; + (as_heap, offset) + }; + self.push_insn(block, Insn::StoreField { recv: ivar_storage, id: id.into(), offset, val }); + self.push_insn(block, Insn::WriteBarrier { recv: self_val, val }); + if next_shape_id != recv_type.shape() { + // Write the new shape ID + let shape_id = self.push_insn(block, Insn::Const { val: Const::CShape(next_shape_id) }); + let shape_id_offset = unsafe { rb_shape_id_offset() }; + self.push_insn(block, Insn::StoreField { recv: self_val, id: FieldName::shape_id, offset: shape_id_offset, val: shape_id }); + } + } _ => { self.push_insn_id(block, insn_id); } } } } - self.infer_types(); + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + fn gen_patch_points_for_optimized_ccall(&mut self, block: BlockId, recv_class: VALUE, method_id: ID, cme: *const rb_callable_method_entry_struct, state: InsnId) { + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state }); + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: recv_class, method: method_id, cme }, state }); + } + + /// Side exit back to the state after a block-backed send. + /// Using the pre-send snapshot would re-execute the send in the interpreter. + fn gen_post_send_no_ep_escape_patch_point(&mut self, block: BlockId, state: &FrameState, insn_idx: u32) { + let iseq = state.iseq; + let mut reload_state = state.clone(); + reload_state.insn_idx = insn_idx as usize; + reload_state.pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + let reload_exit_id = self.push_insn(block, Insn::Snapshot { state: reload_state.without_locals() }); + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: reload_exit_id }); + } + + fn count_not_inlined_cfunc(&mut self, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_inlined_cfunc_counter_pointers = ZJITState::get_not_inlined_cfunc_counter_pointers(); + let counter_ptr = not_inlined_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + self.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); } - /// Optimize SendWithoutBlock that land in a C method to a direct CCall without + fn count_iseq_calls(&mut self, block: BlockId) { + let iseq_name = iseq_get_location(self.iseq, 0); + let access_counter_ptrs = crate::state::ZJITState::get_iseq_calls_count_pointers(); + let counter_ptr = access_counter_ptrs.entry(iseq_name.to_string()).or_insert_with(|| Box::new(0)); + let counter_ptr: &mut u64 = counter_ptr.as_mut(); + + self.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + fn count_not_annotated_cfunc(&mut self, block: BlockId, cme: *const rb_callable_method_entry_t) { + let owner = unsafe { (*cme).owner }; + let called_id = unsafe { (*cme).called_id }; + let qualified_method_name = qualified_method_name(owner, called_id); + let not_annotated_cfunc_counter_pointers = ZJITState::get_not_annotated_cfunc_counter_pointers(); + let counter_ptr = not_annotated_cfunc_counter_pointers.entry(qualified_method_name.clone()).or_insert_with(|| Box::new(0)); + let counter_ptr = &mut **counter_ptr as *mut u64; + + self.push_insn(block, Insn::IncrCounterPtr { counter_ptr }); + } + + /// Optimize Send that land in a C method to a direct CCall without /// runtime lookup. fn optimize_c_calls(&mut self) { - // Try to reduce one SendWithoutBlock to a CCall - fn reduce_to_ccall( + if unsafe { rb_zjit_method_tracing_currently_enabled() } { + return; + } + + // Try to reduce a Send insn to a CCallWithFrame + fn reduce_send_to_ccall( fun: &mut Function, block: BlockId, self_type: Type, send: Insn, send_insn_id: InsnId, ) -> Result<(), ()> { - let Insn::SendWithoutBlock { mut self_val, cd, mut args, state, .. } = send else { + let Insn::Send { mut recv, cd, block: send_block, args, state, .. } = send else { return Err(()); }; @@ -1737,29 +4688,79 @@ impl Function { let method_id = unsafe { rb_vm_ci_mid(call_info) }; // If we have info about the class of the receiver - let (recv_class, profiled_type) = if let Some(class) = self_type.runtime_exact_ruby_class() { - (class, None) - } else { - let iseq_insn_idx = fun.frame_state(state).insn_idx; - let Some(recv_type) = fun.profiled_type_of_at(self_val, iseq_insn_idx) else { return Err(()) }; - (recv_type.class(), Some(recv_type)) + let iseq_insn_idx = fun.frame_state(state).insn_idx; + let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) { + ReceiverTypeResolution::StaticallyKnown { class } => (class, None), + ReceiverTypeResolution::Monomorphic { profiled_type } + | ReceiverTypeResolution::SkewedPolymorphic { profiled_type} => (profiled_type.class(), Some(profiled_type)), + ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()), }; // Do method lookup - let method = unsafe { rb_callable_method_entry(recv_class, method_id) }; - if method.is_null() { + let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) }; + if cme.is_null() { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedMethodType(MethodType::Null)); return Err(()); } // Filter for C methods - let def_type = unsafe { get_cme_def_type(method) }; + let mut def_type = unsafe { get_cme_def_type(cme) }; + while def_type == VM_METHOD_TYPE_ALIAS { + cme = unsafe { rb_aliased_callable_method_entry(cme) }; + def_type = unsafe { get_cme_def_type(cme) }; + } if def_type != VM_METHOD_TYPE_CFUNC { return Err(()); } + + let ci_flags = unsafe { vm_ci_flag(call_info) }; + let visibility = unsafe { METHOD_ENTRY_VISI(cme) }; + match (visibility, ci_flags & VM_CALL_FCALL != 0) { + (METHOD_VISI_PUBLIC, _) => {} + (METHOD_VISI_PRIVATE, true) => {} + (METHOD_VISI_PROTECTED, true) => {} + _ => { + fun.set_dynamic_send_reason(send_insn_id, SendNotOptimizedNeedPermission); + return Err(()); + } + } + + // When seeing &block argument, fall back to dynamic dispatch for now + // TODO: Support block forwarding + if unspecializable_c_call_type(ci_flags) { + // Only count features NOT already counted in type_specialize. + if !unspecializable_call_type(ci_flags) { + fun.count_complex_call_features(block, ci_flags); + } + fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass); + return Err(()); + } + + let blockiseq = match send_block { + Some(BlockHandler::BlockArg) => unreachable!("unsupported &block should have been filtered out"), + Some(BlockHandler::BlockIseq(blockiseq)) => Some(blockiseq), + None => None, + }; + + let cfunc = unsafe { get_cme_def_body_cfunc(cme) }; // Find the `argc` (arity) of the C method, which describes the parameters it expects - let cfunc = unsafe { get_cme_def_body_cfunc(method) }; let cfunc_argc = unsafe { get_mct_argc(cfunc) }; + let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast(); + let name = unsafe { (*cme).called_id }; + + // Look up annotations + let props = ZJITState::get_method_annotations().get_cfunc_properties(cme); + if props.is_none() && get_option!(stats) { + fun.count_not_annotated_cfunc(block, cme); + } + let props = props.unwrap_or_default(); + let return_type = props.return_type; + let elidable = match blockiseq { + Some(_) => false, // Don't consider cfuncs with block arguments as elidable for now + None => props.elidable, + }; + match cfunc_argc { 0.. => { // (self, arg0, arg1, ..., argc) form @@ -1769,59 +4770,289 @@ impl Function { return Err(()); } - // Filter for a leaf and GC free function - use crate::cruby_methods::FnProperties; - let Some(FnProperties { leaf: true, no_gc: true, return_type, elidable }) = - ZJITState::get_method_annotations().get_cfunc_properties(method) - else { + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); return Err(()); - }; + } - let ci_flags = unsafe { vm_ci_flag(call_info) }; - // Filter for simple call sites (i.e. no splats etc.) - if ci_flags & VM_CALL_ARGS_SIMPLE != 0 { - // Commit to the replacement. Put PatchPoint. - fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass: recv_class, method: method_id, cme: method }, state }); - if let Some(profiled_type) = profiled_type { - // Guard receiver class - self_val = fun.push_insn(block, Insn::GuardType { val: self_val, guard_type: Type::from_profiled_type(profiled_type), state }); + // Commit to the replacement. Put PatchPoint. + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + let argc = unsafe { vm_ci_argc(call_info) } as i32; + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend { argc }) }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Try inlining the cfunc into HIR. Only inline if we don't have a block argument + if blockiseq.is_none() { + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.count(block, Counter::inline_cfunc_optimized_send_count); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); } - let cfun = unsafe { get_mct_func(cfunc) }.cast(); - let mut cfunc_args = vec![self_val]; - cfunc_args.append(&mut args); - let ccall = fun.push_insn(block, Insn::CCall { cfun, args: cfunc_args, name: method_id, return_type, elidable }); - fun.make_equal_to(send_insn_id, ccall); - return Ok(()); + + // Only allow leaf calls if we don't have a block argument + if props.leaf && props.no_gc { + fun.count(block, Counter::inline_cfunc_optimized_send_count); + let owner = unsafe { (*cme).owner }; + let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + return Ok(()); + } + } + + // Emit a call + if get_option!(stats) { + fun.count_not_inlined_cfunc(block, cme); } + let ccall = fun.push_insn(block, Insn::CCallWithFrame { + cd, + cfunc: cfunc_ptr, + recv, + args, + cme, + name, + state, + return_type, + elidable, + block: blockiseq.map(BlockHandler::BlockIseq), + }); + fun.make_equal_to(send_insn_id, ccall); + Ok(()) } + // Variadic method -1 => { - // (argc, argv, self) parameter form - // Falling through for now + // The method gets a pointer to the first argument + // func(int argc, VALUE *argv, VALUE recv) + + // Check singleton class assumption first, before emitting other patchpoints + if !fun.assume_no_singleton_classes(block, recv_class, state) { + fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen); + return Err(()); + } + + fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state); + + if let Some(profiled_type) = profiled_type { + // Guard receiver class + let argc = unsafe { vm_ci_argc(call_info) } as i32; + recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile::ProfileSend { argc }) }); + fun.insn_types[recv.0] = fun.infer_type(recv); + } + + // Try inlining the cfunc into HIR. Only inline if we don't have a block argument + if blockiseq.is_none() { + let tmp_block = fun.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns); + fun.blocks[block.0].insns.extend(insns); + fun.count(block, Counter::inline_cfunc_optimized_send_count); + fun.make_equal_to(send_insn_id, replacement); + if fun.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + fun.insn_types[replacement.0] = fun.infer_type(replacement); + } + fun.remove_block(tmp_block); + return Ok(()); + } + + // Only allow leaf calls if we don't have a block argument + if props.leaf && props.no_gc { + fun.count(block, Counter::inline_cfunc_optimized_send_count); + let owner = unsafe { (*cme).owner }; + let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable }); + fun.make_equal_to(send_insn_id, ccall); + return Ok(()); + } + } + + // No inlining; emit a call + if get_option!(stats) { + fun.count_not_inlined_cfunc(block, cme); + } + + let ccall = fun.push_insn(block, Insn::CCallVariadic { + cfunc: cfunc_ptr, + recv, + args, + cme, + name: method_id, + state, + return_type, + elidable, + block: blockiseq.map(BlockHandler::BlockIseq), + }); + + fun.make_equal_to(send_insn_id, ccall); + Ok(()) } -2 => { - // (self, args_ruby_array) parameter form - // Falling through for now + // (self, args_ruby_array) + fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic); + Err(()) } _ => unreachable!("unknown cfunc kind: argc={argc}") } - - Err(()) } - for block in self.rpo() { + for block in self.reverse_post_order() { let old_insns = std::mem::take(&mut self.blocks[block.0].insns); assert!(self.blocks[block.0].insns.is_empty()); for insn_id in old_insns { - if let send @ Insn::SendWithoutBlock { self_val, .. } = self.find(insn_id) { - let self_type = self.type_of(self_val); - if reduce_to_ccall(self, block, self_type, send, insn_id).is_ok() { - continue; + let send = self.find(insn_id); + match send { + send @ Insn::Send { recv, .. } => { + let recv_type = self.type_of(recv); + if reduce_send_to_ccall(self, block, recv_type, send, insn_id).is_ok() { + continue; + } + } + Insn::InvokeBuiltin { bf, recv, args, state, .. } => { + let props = ZJITState::get_method_annotations().get_builtin_properties(&bf).unwrap_or_default(); + // Try inlining the cfunc into HIR + let tmp_block = self.new_block(u32::MAX); + if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { + // Copy contents of tmp_block to block + assert_ne!(block, tmp_block); + let insns = std::mem::take(&mut self.blocks[tmp_block.0].insns); + self.blocks[block.0].insns.extend(insns); + self.count(block, Counter::inline_cfunc_optimized_send_count); + self.make_equal_to(insn_id, replacement); + if self.type_of(replacement).bit_equal(types::Any) { + // Not set yet; infer type + self.insn_types[replacement.0] = self.infer_type(replacement); + } + self.remove_block(tmp_block); + continue; + } } + _ => {} } self.push_insn_id(block, insn_id); } } - self.infer_types(); + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + + /// Convert `Send` instructions with no profile data into `SideExit` with recompile info. + /// This runs after strength reduction passes (type_specialize, inline, optimize_c_calls) so + /// that sends that can be optimized without profiling (e.g. known CFUNCs) are already handled. + /// The remaining no-profile sends are turned into side exits that trigger recompilation with + /// fresh profile data. + fn convert_no_profile_sends(&mut self) { + // On the final version, recompilation is not possible, so converting sends to + // SideExits would just add overhead (the exit fires every time without benefit). + // Keep them as Send fallbacks so the interpreter handles them directly. + let payload = get_or_create_iseq_payload(self.iseq); + if payload.versions.len() + 1 >= crate::codegen::max_iseq_versions() { + return; + } + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + assert!(self.blocks[block.0].insns.is_empty()); + for insn_id in old_insns { + match self.find(insn_id) { + Insn::Send { cd, state, reason: SendFallbackReason::SendWithoutBlockNoProfiles | SendFallbackReason::SendNoProfiles, .. } => { + let argc = unsafe { vm_ci_argc((*cd).ci) } as i32; + self.push_insn(block, Insn::SideExit { state, reason: SideExitReason::NoProfileSend, recompile: Some(Recompile::ProfileSend { argc }) }); + // SideExit is a terminator; don't add remaining instructions + break; + } + _ => { + self.push_insn_id(block, insn_id); + } + } + } + } + } + + fn optimize_load_store(&mut self) { + for block in self.reverse_post_order() { + let mut compile_time_heap: HashMap<(InsnId, i32), InsnId> = HashMap::new(); + let old_insns = std::mem::take(&mut self.blocks[block.0].insns); + let mut new_insns = vec![]; + for insn_id in old_insns { + let replacement_insn: InsnId = match self.find(insn_id) { + Insn::StoreField { recv, offset, val, .. } => { + let key = (self.chase_insn(recv), offset); + let heap_entry = compile_time_heap.get(&key).copied(); + // TODO(Jacob): Switch from actual to partial equality + if Some(val) == heap_entry { + // If the value is already stored, short circuit and don't add an instruction to the block + continue + } + // TODO(Jacob): Add TBAA to avoid removing so many entries + compile_time_heap.retain(|(_, off), _| *off != offset); + compile_time_heap.insert(key, val); + insn_id + }, + Insn::LoadField { recv, offset, return_type, .. } => { + let key = (self.chase_insn(recv), offset); + match compile_time_heap.entry(key) { + std::collections::hash_map::Entry::Occupied(entry) => { + let cached_insn = *entry.get(); + + // TODO (nirvdrum 2026-06-04): Remove the return type guard and supporting code when the type checker becomes more accurate. + // If there's an an embedded<=>heap shape storage transition, it's possible for this `LoadField` to have a different return + // type than the cached entry (`CPtr` vs `BasicObject`). While the loaded value would be the same in either case, the + // difference in associated type causes type checking to fail. Consequently, we conservatively retain the duplicate `LoadField`. + // The `optimize_load_store_does_not_alias_loads_with_incompatible_return_types` test checks the problematic case. + let can_forward_cached_insn = match self.find(cached_insn) { + Insn::LoadField { return_type : cached_return_type,.. } => cached_return_type.is_subtype(return_type), + _ => true + }; + + if can_forward_cached_insn { + // If the value is stored already, we should short circuit. + // However, we need to replace insn_id with its representative in the SSA union. + self.make_equal_to(insn_id, cached_insn); + continue + } + } + std::collections::hash_map::Entry::Vacant(_) => { + // If the value has not been accessed, cache a copy to optimize future loads or stores. + compile_time_heap.insert(key, insn_id); + } + } + insn_id + } + Insn::WriteBarrier { .. } => { + // Currently, WriteBarrier write effects are Allocator and Memory when we'd really like them to be flags. + // We don't use LoadField for mark bits so we can ignore them for now. + // But flags does not exist in our effects abstract heap modeling and we don't want to add special casing to effects. + // This special casing in this pass here should be removed once we refine our effects system to provide greater granularity for WriteBarrier. + // TODO: use TBAA + let offset = RUBY_OFFSET_RBASIC_FLAGS; + compile_time_heap.retain(|(_, off), _| *off != offset); + insn_id + }, + insn => { + // If an instruction affects memory and we haven't modeled it, the compile_time_heap is invalidated + if insn.effects_of().includes(Effect::write(abstract_heaps::Memory)) { + compile_time_heap.clear(); + } + insn_id + } + }; + new_insns.push(replacement_insn); + } + self.blocks[block.0].insns = new_insns; + } } /// Fold a binary operator on fixnums. @@ -1840,6 +5071,56 @@ impl Function { .unwrap_or(insn_id) } + /// Block-local canonicalize: rewrite each operand through union-find and a + /// per-block map of the most recent `Guard*` for that value. Forwards + /// guarded values into branch-edge args (so `infer_types` narrows merge-block + /// parameters and `fold_constants` drops redundant CFG-join guards) and + /// ordinary in-block uses. + /// + /// `Guard*` substitutions are unconditional within a block: a guard's + /// side-exit semantics guarantee the substituted value type holds for every + /// downstream use in the same block. + /// + /// `RefineType` is intentionally skipped: its narrowing is only valid on one + /// branch arm, which would require dropping refine-derived rewrites at each + /// `IfTrue`/`IfFalse`. Cross-arm refine forwarding is left for a follow-up + /// dominator-scoped pass. + /// + /// Inspired by Cranelift's aegraph canonicalize step + /// (<https://cfallin.org/blog/2026/04/09/aegraph/>). + fn canonicalize(&mut self) { + let mut rewrite_map: HashMap<InsnId, InsnId> = HashMap::new(); + for block in self.reverse_post_order() { + rewrite_map.clear(); + for i in 0..self.blocks[block.0].insns.len() { + let insn_id = self.blocks[block.0].insns[i]; + let canonical_id = self.union_find.borrow().find_const(insn_id); + + let union_find = &self.union_find; + self.insns[canonical_id.0].for_each_operand_mut(|operand| { + let canon = union_find.borrow().find_const(*operand); + *operand = rewrite_map.get(&canon).copied().unwrap_or(canon); + }); + + // For the binary guards only `left` is registered because their infer_type is + // type_of(left). + match &self.insns[canonical_id.0] { + Insn::GuardType { val: src, .. } + | Insn::GuardBitEquals { val: src, .. } + | Insn::GuardAnyBitSet { val: src, .. } + | Insn::GuardNoBitsSet { val: src, .. } + | Insn::GuardGreaterEq { left: src, .. } + | Insn::GuardLess { left: src, .. } => { + rewrite_map.insert(*src, canonical_id); + } + _ => {} + } + } + } + + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + /// Use type information left by `infer_types` to fold away operations that can be evaluated at compile-time. /// /// It can fold fixnum math, truthiness tests, and branches with constant conditionals. @@ -1850,7 +5131,7 @@ impl Function { // // This would require 1) fixpointing, 2) worklist, or 3) (slightly less powerful) calling a // function-level infer_types after each pruned branch. - for block in self.rpo() { + for block in self.reverse_post_order() { let old_insns = std::mem::take(&mut self.blocks[block.0].insns); let mut new_insns = vec![]; for insn_id in old_insns { @@ -1860,25 +5141,207 @@ impl Function { // Don't bother re-inferring the type of val; we already know it. continue; } + Insn::RefineType { val, new_type, .. } if self.is_a(val, new_type) => { + self.make_equal_to(insn_id, val); + // Don't bother re-inferring the type of val; we already know it. + continue; + } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::BasicObject) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const VALUE; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + Insn::LoadField { recv, offset, return_type, .. } if return_type.is_subtype(types::CShape) && + u32::try_from(offset).is_ok() => { + let offset = (offset as u32).to_usize(); + let recv_type = self.type_of(recv); + match recv_type.ruby_object() { + Some(recv_obj) if recv_obj.is_frozen() => { + let recv_ptr = recv_obj.as_ptr() as *const u32; + let val = unsafe { recv_ptr.byte_add(offset).read() }; + self.new_insn(Insn::Const { val: Const::CShape(ShapeId(val)) }) + } + _ => insn_id, + } + } + Insn::ArrayLength { array } => { + match self.type_of(array).ruby_object() { + Some(array_obj) if array_obj.is_frozen() => { + let length = unsafe { rb_jit_array_len(array_obj) }; + self.new_insn(Insn::Const { val: Const::CInt64(length) }) + } + _ => insn_id, + } + } + Insn::UnboxFixnum { val } => { + let recv_type = self.type_of(val); + match recv_type.fixnum_value() { + Some(val) => self.new_insn(Insn::Const { val: Const::CInt64(val) }), + _ => insn_id, + } + }, + Insn::GuardGreaterEq { left, right, state, reason } => { + let left_num = self.type_of(left).cint64_value(); + let right_num = self.type_of(right).cint64_value(); + match (left_num, right_num) { + (Some(l), Some(r)) if l >= r => { + self.make_equal_to(insn_id, left); + continue + }, + (Some(_), Some(_)) => self.new_insn(Insn::SideExit { state, reason, recompile: None }), + _ => insn_id, + } + }, + Insn::GuardLess { left, right, state, reason } => { + let left_num = self.type_of(left).cint64_value(); + let right_num = self.type_of(right).cint64_value(); + match (left_num, right_num) { + (Some(l), Some(r)) if l < r => { + self.make_equal_to(insn_id, left); + continue + }, + (Some(_), Some(_)) => self.new_insn(Insn::SideExit { state, reason, recompile: None }), + _ => insn_id, + } + }, + Insn::GuardBitEquals { val, expected, .. } => { + let recv_type = self.type_of(val); + if recv_type.has_value(expected) { + continue; + } else { + insn_id + } + } + Insn::AnyToString { str, .. } if self.is_a(str, types::String) => { + self.make_equal_to(insn_id, str); + // Don't bother re-inferring the type of str; we already know it. + continue; + } + Insn::IsA { val, class } => 'is_a: { + let class_type = self.type_of(class); + if !class_type.is_subtype(types::Class) { + break 'is_a insn_id; + } + let Some(class_value) = class_type.ruby_object() else { + break 'is_a insn_id; + }; + let val_type = self.type_of(val); + let the_class = Type::from_class_inexact(class_value); + if val_type.is_subtype(the_class) { + self.new_insn(Insn::Const { val: Const::Value(Qtrue) }) + } else if !val_type.could_be(the_class) { + self.new_insn(Insn::Const { val: Const::Value(Qfalse) }) + } else { + insn_id + } + } + Insn::StringEqual { left, right } => { + let left = self.chase_insn(left); + let right = self.chase_insn(right); + // If both operands resolve to the same SSA value, + // String#== is guaranteed to be true. + if left == right { + self.new_insn(Insn::Const { val: Const::Value(Qtrue) }) + } else { + let left_type = self.type_of(left); + let right_type = self.type_of(right); + match (left_type.ruby_object(), right_type.ruby_object()) { + (Some(left_obj), Some(right_obj)) + if left_obj.is_frozen() && right_obj.is_frozen() => + { + // For known frozen objects, evaluate String#== at compile time. + let val = unsafe { rb_yarv_str_eql_internal(left_obj, right_obj) }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + } Insn::FixnumAdd { left, right, .. } => { + match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { + (Some(0), _) => { self.make_equal_to(insn_id, right); continue; } + (_, Some(0)) => { self.make_equal_to(insn_id, left); continue; } + _ => {} + } self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { (Some(l), Some(r)) => l.checked_add(r), _ => None, }) } Insn::FixnumSub { left, right, .. } => { + match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { + (_, Some(0)) => { self.make_equal_to(insn_id, left); continue; } + _ => {} + } self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { (Some(l), Some(r)) => l.checked_sub(r), _ => None, }) } Insn::FixnumMult { left, right, .. } => { + match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { + (Some(1), _) => { self.make_equal_to(insn_id, right); continue; } + (_, Some(1)) => { self.make_equal_to(insn_id, left); continue; } + _ => {} + } self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { (Some(l), Some(r)) => l.checked_mul(r), (Some(0), _) | (_, Some(0)) => Some(0), _ => None, }) } + Insn::FixnumDiv { left, right, .. } => { + match (self.type_of(left).fixnum_value(), self.type_of(right).fixnum_value()) { + (_, Some(1)) => { self.make_equal_to(insn_id, left); continue; } + _ => {} + } + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) if l == (RUBY_FIXNUM_MIN as i64) && r == -1 => None, // Avoid Fixnum overflow + (Some(_l), Some(r)) if r == 0 => None, // Avoid Divide by zero. + (Some(l), Some(r)) => { + let l_obj = VALUE::fixnum_from_isize(l as isize); + let r_obj = VALUE::fixnum_from_isize(r as isize); + Some(unsafe { rb_jit_fix_div_fix(l_obj, r_obj) }.as_fixnum()) + }, + _ => None, + }) + } + Insn::FixnumMod { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) if r != 0 => { + let l_obj = VALUE::fixnum_from_isize(l as isize); + let r_obj = VALUE::fixnum_from_isize(r as isize); + Some(unsafe { rb_jit_fix_mod_fix(l_obj, r_obj) }.as_fixnum()) + }, + _ => None, + }) + } + Insn::FixnumXor { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l ^ r), + _ => None, + }) + } + Insn::FixnumAnd { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l & r), + _ => None, + }) + } + Insn::FixnumOr { left, right, .. } => { + self.fold_fixnum_bop(insn_id, left, right, |l, r| match (l, r) { + (Some(l), Some(r)) => Some(l | r), + _ => None, + }) + } Insn::FixnumEq { left, right, .. } => { self.fold_fixnum_pred(insn_id, left, right, |l, r| match (l, r) { (Some(l), Some(r)) => Some(l == r), @@ -1915,31 +5378,55 @@ impl Function { _ => None, }) } + Insn::ArrayAref { array, index } + if self.type_of(array).ruby_object_known() + && self.type_of(index).is_subtype(types::CInt64) => { + let array_obj = self.type_of(array).ruby_object().unwrap(); + match (array_obj.is_frozen(), self.type_of(index).cint64_value()) { + (true, Some(index)) => { + let val = unsafe { rb_yarv_ary_entry_internal(array_obj, index) }; + self.new_insn(Insn::Const { val: Const::Value(val) }) + } + _ => insn_id, + } + } + Insn::AdjustBounds { index, .. } => { + // If index is known nonnegative, then we don't need to adjust bounds. + if self.type_of(index).known_nonnegative() { + self.make_equal_to(insn_id, index); + // Don't bother re-inferring the type of index; we already know it. + continue; + } else { + insn_id + } + } Insn::Test { val } if self.type_of(val).is_known_falsy() => { self.new_insn(Insn::Const { val: Const::CBool(false) }) } Insn::Test { val } if self.type_of(val).is_known_truthy() => { self.new_insn(Insn::Const { val: Const::CBool(true) }) } - Insn::IfTrue { val, target } if self.is_a(val, Type::from_cbool(true)) => { - self.new_insn(Insn::Jump(target)) + Insn::Test { val: test_val } => { + if let Insn::BoxBool { val: bool_val } = self.find(test_val) { + self.make_equal_to(insn_id, bool_val); + continue; + } else { + insn_id + } + } + Insn::CondBranch { val, if_true, .. } if self.is_a(val, Type::from_cbool(true)) => { + self.new_insn(Insn::Jump(if_true)) } - Insn::IfFalse { val, target } if self.is_a(val, Type::from_cbool(false)) => { - self.new_insn(Insn::Jump(target)) + Insn::CondBranch { val, if_false, .. } if self.is_a(val, Type::from_cbool(false)) => { + self.new_insn(Insn::Jump(if_false)) } - // If we know that the branch condition is never going to cause a branch, - // completely drop the branch from the block. - Insn::IfTrue { val, .. } if self.is_a(val, Type::from_cbool(false)) => continue, - Insn::IfFalse { val, .. } if self.is_a(val, Type::from_cbool(true)) => continue, _ => insn_id, }; // If we're adding a new instruction, mark the two equivalent in the union-find and // do an incremental flow typing of the new instruction. - if insn_id != replacement_id { + if insn_id != replacement_id && self.insns[replacement_id.0].has_output() { self.make_equal_to(insn_id, replacement_id); - if self.insns[replacement_id.0].has_output() { - self.insn_types[replacement_id.0] = self.infer_type(replacement_id); - } + self.insn_types[replacement_id.0] = self.infer_type(replacement_id); } new_insns.push(replacement_id); // If we've just folded an IfTrue into a Jump, for example, don't bother copying @@ -1952,150 +5439,16 @@ impl Function { } } - fn worklist_traverse_single_insn(&self, insn: &Insn, worklist: &mut VecDeque<InsnId>) { - match insn { - &Insn::Const { .. } - | &Insn::Param { .. } - | &Insn::GetLocal { .. } - | &Insn::PutSpecialObject { .. } - | &Insn::IncrCounter(_) => - {} - &Insn::PatchPoint { state, .. } - | &Insn::CheckInterrupts { state } - | &Insn::GetConstantPath { ic: _, state } => { - worklist.push_back(state); - } - &Insn::ArrayMax { ref elements, state } - | &Insn::NewArray { ref elements, state } => { - worklist.extend(elements); - worklist.push_back(state); - } - &Insn::NewHash { ref elements, state } => { - for &(key, value) in elements { - worklist.push_back(key); - worklist.push_back(value); - } - worklist.push_back(state); - } - &Insn::NewRange { low, high, state, .. } => { - worklist.push_back(low); - worklist.push_back(high); - worklist.push_back(state); - } - &Insn::StringConcat { ref strings, state, .. } => { - worklist.extend(strings); - worklist.push_back(state); - } - &Insn::ToRegexp { ref values, state, .. } => { - worklist.extend(values); - worklist.push_back(state); - } - | &Insn::Return { val } - | &Insn::Throw { val, .. } - | &Insn::Test { val } - | &Insn::SetLocal { val, .. } - | &Insn::IsNil { val } => - worklist.push_back(val), - &Insn::SetGlobal { val, state, .. } - | &Insn::Defined { v: val, state, .. } - | &Insn::StringIntern { val, state } - | &Insn::StringCopy { val, state, .. } - | &Insn::GuardType { val, state, .. } - | &Insn::GuardBitEquals { val, state, .. } - | &Insn::ToArray { val, state } - | &Insn::ToNewArray { val, state } => { - worklist.push_back(val); - worklist.push_back(state); - } - &Insn::Snapshot { ref state } => { - worklist.extend(&state.stack); - worklist.extend(&state.locals); - } - &Insn::FixnumAdd { left, right, state } - | &Insn::FixnumSub { left, right, state } - | &Insn::FixnumMult { left, right, state } - | &Insn::FixnumDiv { left, right, state } - | &Insn::FixnumMod { left, right, state } - | &Insn::ArrayExtend { left, right, state } - => { - worklist.push_back(left); - worklist.push_back(right); - worklist.push_back(state); - } - &Insn::FixnumLt { left, right } - | &Insn::FixnumLe { left, right } - | &Insn::FixnumGt { left, right } - | &Insn::FixnumGe { left, right } - | &Insn::FixnumEq { left, right } - | &Insn::FixnumNeq { left, right } - | &Insn::FixnumAnd { left, right } - | &Insn::FixnumOr { left, right } - => { - worklist.push_back(left); - worklist.push_back(right); - } - &Insn::Jump(BranchEdge { ref args, .. }) => worklist.extend(args), - &Insn::IfTrue { val, target: BranchEdge { ref args, .. } } | &Insn::IfFalse { val, target: BranchEdge { ref args, .. } } => { - worklist.push_back(val); - worklist.extend(args); - } - &Insn::ArrayDup { val, state } | &Insn::HashDup { val, state } => { - worklist.push_back(val); - worklist.push_back(state); - } - &Insn::Send { self_val, ref args, state, .. } - | &Insn::SendWithoutBlock { self_val, ref args, state, .. } - | &Insn::SendWithoutBlockDirect { self_val, ref args, state, .. } => { - worklist.push_back(self_val); - worklist.extend(args); - worklist.push_back(state); - } - &Insn::InvokeBuiltin { ref args, state, .. } => { - worklist.extend(args); - worklist.push_back(state) - } - &Insn::CCall { ref args, .. } => worklist.extend(args), - &Insn::GetIvar { self_val, state, .. } | &Insn::DefinedIvar { self_val, state, .. } => { - worklist.push_back(self_val); - worklist.push_back(state); - } - &Insn::SetIvar { self_val, val, state, .. } => { - worklist.push_back(self_val); - worklist.push_back(val); - worklist.push_back(state); - } - &Insn::ArrayPush { array, val, state } => { - worklist.push_back(array); - worklist.push_back(val); - worklist.push_back(state); - } - &Insn::ObjToString { val, state, .. } => { - worklist.push_back(val); - worklist.push_back(state); - } - &Insn::AnyToString { val, str, state, .. } => { - worklist.push_back(val); - worklist.push_back(str); - worklist.push_back(state); - } - &Insn::GetGlobal { state, .. } | - &Insn::GetSpecialSymbol { state, .. } | - &Insn::GetSpecialNumber { state, .. } | - &Insn::SideExit { state, .. } => worklist.push_back(state), - } - } - /// Remove instructions that do not have side effects and are not referenced by any other /// instruction. fn eliminate_dead_code(&mut self) { - let rpo = self.rpo(); + let rpo = self.reverse_post_order(); let mut worklist = VecDeque::new(); // Find all of the instructions that have side effects, are control instructions, or are // otherwise necessary to keep around for block_id in &rpo { for insn_id in &self.blocks[block_id.0].insns { - let insn = &self.insns[insn_id.0]; - if insn.has_effects() { + if !&self.insns[insn_id.0].is_elidable() { worklist.push_back(*insn_id); } } @@ -2105,7 +5458,10 @@ impl Function { while let Some(insn_id) = worklist.pop_front() { if necessary.get(insn_id) { continue; } necessary.insert(insn_id); - self.worklist_traverse_single_insn(&self.find(insn_id), &mut worklist); + let insn_id = self.union_find.borrow().find_const(insn_id); + self.insns[insn_id.0].for_each_operand(|operand| { + worklist.push_back(self.union_find.borrow().find_const(operand)); + }); } // Now remove all unnecessary instructions for block_id in &rpo { @@ -2113,7 +5469,7 @@ impl Function { } } - fn absorb_dst_block(&mut self, num_in_edges: &Vec<u32>, block: BlockId) -> bool { + fn absorb_dst_block(&mut self, num_in_edges: &[u32], block: BlockId) -> bool { let Some(terminator_id) = self.blocks[block.0].insns.last() else { return false }; let Insn::Jump(BranchEdge { target, args }) = self.find(*terminator_id) @@ -2147,17 +5503,15 @@ impl Function { // * blocks that get absorbed are not in RPO anymore // * blocks pointed to by blocks that get absorbed retain the same number of in-edges let mut num_in_edges = vec![0; self.blocks.len()]; - for block in self.rpo() { - for &insn in &self.blocks[block.0].insns { - if let Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } | Insn::Jump(target) = self.find(insn) { - num_in_edges[target.target.0] += 1; - } + for block in self.reverse_post_order() { + for target in self.successors(block) { + num_in_edges[target.0] += 1; } } let mut changed = false; loop { let mut iter_changed = false; - for block in self.rpo() { + for block in self.reverse_post_order() { // Ignore transient empty blocks if self.blocks[block.0].insns.is_empty() { continue; } loop { @@ -2170,13 +5524,76 @@ impl Function { changed = true; } if changed { - self.infer_types(); + crate::stats::trace_compile_phase("infer_types", || self.infer_types()); + } + } + + /// Remove duplicate PatchPoint instructions within each basic block. + /// Two PatchPoints are redundant if they assert the same Invariant and no + /// intervening instruction could invalidate it (i.e., writes to PatchPoint). + fn remove_redundant_patch_points(&mut self) { + for block_id in self.reverse_post_order() { + let mut seen = HashSet::new(); + let insns = std::mem::take(&mut self.blocks[block_id.0].insns); + let mut new_insns = Vec::with_capacity(insns.len()); + for insn_id in insns { + let insn = self.find(insn_id); + if let Insn::PatchPoint { invariant, .. } = insn { + if !seen.insert(invariant) { + continue; + } + } else if insn.effects_of().write_bits().overlaps(abstract_heaps::PatchPoint) { + seen.clear(); + } + new_insns.push(insn_id); + } + self.blocks[block_id.0].insns = new_insns; + } + } + + /// Remove duplicate CheckInterrupts instructions within each basic block. + /// Only the first CheckInterrupts in a block is needed unless an intervening + /// instruction writes to InterruptFlag (e.g. a call), which resets tracking. + fn remove_duplicate_check_interrupts(&mut self) { + for block_id in self.reverse_post_order() { + let mut seen = false; + let insns = std::mem::take(&mut self.blocks[block_id.0].insns); + let mut new_insns = Vec::with_capacity(insns.len()); + for insn_id in insns { + let insn = &self.insns[insn_id.0]; + if matches!(insn, Insn::CheckInterrupts { .. }) { + if seen { continue; } + seen = true; + } else if insn.effects_of().write_bits().overlaps(abstract_heaps::InterruptFlag) { + seen = false; + } + new_insns.push(insn_id); + } + self.blocks[block_id.0].insns = new_insns; } } + /// Return a list that has entry_block and then jit_entry_blocks + fn entry_blocks(&self) -> Vec<BlockId> { + let mut entry_blocks = self.jit_entry_blocks.clone(); + entry_blocks.insert(0, self.entry_block); + entry_blocks + } + + pub fn is_entry_block(&self, block_id: BlockId) -> bool { + self.entry_block == block_id || self.jit_entry_blocks.contains(&block_id) + } + + /// Populate the entries superblock with an Entries instruction targeting all entry blocks. + /// Must be called after all entry blocks have been created. + fn seal_entries(&mut self) { + let targets = self.entry_blocks(); + self.push_insn(self.entries_block, Insn::Entries { targets }); + } + /// Return a traversal of the `Function`'s `BlockId`s in reverse post-order. - pub fn rpo(&self) -> Vec<BlockId> { - let mut result = self.po_from(self.entry_block); + pub fn reverse_post_order(&self) -> Vec<BlockId> { + let mut result = self.po_from(self.entries_block); result.reverse(); result } @@ -2197,11 +5614,8 @@ impl Function { } if !seen.insert(block) { continue; } stack.push((block, Action::VisitSelf)); - for insn_id in &self.blocks[block.0].insns { - let insn = self.find(*insn_id); - if let Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } | Insn::Jump(target) = insn { - stack.push((target.target, Action::VisitEdges)); - } + for target in self.successors(block) { + stack.push((target, Action::VisitEdges)); } } result @@ -2216,34 +5630,234 @@ impl Function { } } + /// Helper function to make an Iongraph JSON "instruction". + /// `uses`, `memInputs` and `attributes` are left empty for now, but may be populated + /// in the future. + fn make_iongraph_instr(id: InsnId, inputs: Vec<Json>, opcode: &str, ty: &str) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("opcode", opcode) + .insert("attributes", Json::empty_array()) + .insert("inputs", Json::Array(inputs)) + .insert("uses", Json::empty_array()) + .insert("memInputs", Json::empty_array()) + .insert("type", ty) + .build() + } + + /// Helper function to make an Iongraph JSON "block". + fn make_iongraph_block(id: BlockId, predecessors: Vec<BlockId>, successors: Vec<BlockId>, instructions: Vec<Json>, attributes: Vec<&str>, loop_depth: u32) -> Json { + Json::object() + // Add an offset of 0x1000 to avoid the `ptr` being 0x0, which iongraph rejects. + .insert("ptr", id.0 + 0x1000) + .insert("id", id.0) + .insert("loopDepth", loop_depth) + .insert("attributes", Json::array(attributes)) + .insert("predecessors", Json::array(predecessors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("successors", Json::array(successors.iter().map(|x| x.0).collect::<Vec<usize>>())) + .insert("instructions", Json::array(instructions)) + .build() + } + + /// Helper function to make an Iongraph JSON "function". + /// Note that `lir` is unpopulated right now as ZJIT doesn't use its functionality. + fn make_iongraph_function(pass_name: &str, hir_blocks: Vec<Json>) -> Json { + Json::object() + .insert("name", pass_name) + .insert("mir", Json::object() + .insert("blocks", Json::array(hir_blocks)) + .build() + ) + .insert("lir", Json::object() + .insert("blocks", Json::empty_array()) + .build() + ) + .build() + } + + /// Generate an iongraph JSON pass representation for this function. + pub fn to_iongraph_pass(&self, pass_name: &str) -> Json { + let mut ptr_map = PtrPrintMap::identity(); + if cfg!(test) { + ptr_map.map_ptrs = true; + } + + let mut hir_blocks = Vec::new(); + let cfi = ControlFlowInfo::new(self); + let dominators = Dominators::new(self); + let loop_info = LoopInfo::new(&cfi, &dominators); + + // Push each block from the iteration in reverse post order to `hir_blocks`. + for block_id in self.reverse_post_order() { + // Create the block with instructions. + let block = &self.blocks[block_id.0]; + let predecessors = cfi.predecessors(block_id).collect(); + let successors = cfi.successors(block_id).collect(); + let mut instructions = Vec::new(); + + // Process all instructions (parameters and body instructions). + // Parameters are currently guaranteed to be Parameter instructions, but in the future + // they might be refined to other instruction kinds by the optimizer. + for insn_id in block.params.iter().chain(block.insns.iter()) { + let insn_id = self.union_find.borrow().find_const(*insn_id); + let insn = self.find(insn_id); + + // Snapshots are not serialized, so skip them. + if matches!(insn, Insn::Snapshot {..}) { + continue; + } + + // Instructions with no output or an empty type should have an empty type field. + let type_str = if insn.has_output() { + let insn_type = self.type_of(insn_id); + if insn_type.is_subtype(types::Empty) { + String::new() + } else { + insn_type.print(&ptr_map).to_string() + } + } else { + String::new() + }; + + + let opcode = insn.print(&ptr_map, Some(self.iseq)).to_string(); + + // Collect inputs for a given instruction. + let mut inputs = Vec::new(); + insn.for_each_operand(|id| inputs.push(id.0.into())); + let inputs: Vec<Json> = inputs; + + instructions.push( + Self::make_iongraph_instr( + insn_id, + inputs, + &opcode, + &type_str + ) + ); + } + + let mut attributes = vec![]; + if loop_info.is_back_edge_source(block_id) { + attributes.push("backedge"); + } + if loop_info.is_loop_header(block_id) { + attributes.push("loopheader"); + } + let loop_depth = loop_info.loop_depth(block_id); + + hir_blocks.push(Self::make_iongraph_block( + block_id, + predecessors, + successors, + instructions, + attributes, + loop_depth, + )); + } + + Self::make_iongraph_function(pass_name, hir_blocks) + } + /// Run all the optimization passes we have. pub fn optimize(&mut self) { + let mut passes: Vec<Json> = Vec::new(); + let should_dump = get_option!(dump_hir_iongraph); + + macro_rules! counter_for { + // Bucket all strength reduction together + (type_specialize) => { Counter::compile_hir_strength_reduce_time_ns }; + (inline) => { Counter::compile_hir_strength_reduce_time_ns }; + (optimize_getivar) => { Counter::compile_hir_strength_reduce_time_ns }; + (optimize_c_calls) => { Counter::compile_hir_strength_reduce_time_ns }; + (convert_no_profile_sends) => { Counter::compile_hir_strength_reduce_time_ns }; + // End strength reduction bucket + (optimize_load_store) => { Counter::compile_hir_optimize_load_store_time_ns }; + (canonicalize) => { Counter::compile_hir_canonicalize_time_ns }; + (fold_constants) => { Counter::compile_hir_fold_constants_time_ns }; + (clean_cfg) => { Counter::compile_hir_clean_cfg_time_ns }; + (remove_redundant_patch_points) => { Counter::compile_hir_remove_redundant_patch_points_time_ns }; + (remove_duplicate_check_interrupts) => { Counter::compile_hir_remove_duplicate_check_interrupts_time_ns }; + (eliminate_dead_code) => { Counter::compile_hir_eliminate_dead_code_time_ns }; + ($name:ident) => { unimplemented!("Counter for pass {}", stringify!($name)) }; + } + + macro_rules! run_pass { + ($name:ident) => { + let counter = counter_for!($name); + crate::stats::trace_compile_phase(stringify!($name), || + crate::stats::with_time_stat(counter, || self.$name()) + ); + #[cfg(debug_assertions)] crate::stats::trace_compile_phase("validate", || self.assert_validates()); + if should_dump { + passes.push( + self.to_iongraph_pass(stringify!($name)) + ); + } + } + } + + if should_dump { + passes.push(self.to_iongraph_pass("unoptimized")); + } + // Function is assumed to have types inferred already - self.optimize_direct_sends(); - #[cfg(debug_assertions)] self.assert_validates(); - self.optimize_c_calls(); - #[cfg(debug_assertions)] self.assert_validates(); - self.fold_constants(); - #[cfg(debug_assertions)] self.assert_validates(); - self.clean_cfg(); - #[cfg(debug_assertions)] self.assert_validates(); - self.eliminate_dead_code(); - #[cfg(debug_assertions)] self.assert_validates(); + run_pass!(type_specialize); + run_pass!(inline); + run_pass!(optimize_getivar); + run_pass!(optimize_c_calls); + run_pass!(convert_no_profile_sends); + run_pass!(optimize_load_store); + run_pass!(canonicalize); + run_pass!(fold_constants); + run_pass!(clean_cfg); + run_pass!(remove_redundant_patch_points); + run_pass!(remove_duplicate_check_interrupts); + run_pass!(eliminate_dead_code); + + if should_dump { + let iseq_name = iseq_get_location(self.iseq, 0); + self.dump_iongraph(&iseq_name, passes); + } } /// Dump HIR passed to codegen if specified by options. pub fn dump_hir(&self) { // Dump HIR after optimization match get_option!(dump_hir_opt) { - Some(DumpHIR::WithoutSnapshot) => println!("Optimized HIR:\n{}", FunctionPrinter::without_snapshot(&self)), - Some(DumpHIR::All) => println!("Optimized HIR:\n{}", FunctionPrinter::with_snapshot(&self)), + Some(DumpHIR::WithoutSnapshot) => println!("Optimized HIR:\n{}", FunctionPrinter::without_snapshot(self)), + Some(DumpHIR::All) => println!("Optimized HIR:\n{}", FunctionPrinter::with_snapshot(self)), Some(DumpHIR::Debug) => println!("Optimized HIR:\n{:#?}", &self), None => {}, } + } - if get_option!(dump_hir_graphviz) { - println!("{}", FunctionGraphvizPrinter::new(&self)); + pub fn dump_iongraph(&self, function_name: &str, passes: Vec<Json>) { + fn sanitize_for_filename(name: &str) -> String { + name.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' || c == '-' { + c + } else { + '_' + } + }) + .collect() } + + use std::io::Write; + let dir = format!("/tmp/zjit-iongraph-{}", std::process::id()); + std::fs::create_dir_all(&dir).expect("Unable to create directory."); + let sanitized = sanitize_for_filename(function_name); + let path = format!("{dir}/func_{sanitized}.json"); + let mut file = std::fs::File::create(path).unwrap(); + let json = Json::object() + .insert("name", function_name) + .insert("passes", passes) + .build(); + writeln!(file, "{json}").unwrap(); } /// Validates the following: @@ -2251,35 +5865,45 @@ impl Function { /// 2. Every terminator must be in the last position. /// 3. Every block must have a terminator. fn validate_block_terminators_and_jumps(&self) -> Result<(), ValidationError> { - for block_id in self.rpo() { - let mut block_has_terminator = false; + let check_edge = |block_id: BlockId, edge: &BranchEdge| -> Result<(), ValidationError> { + let target_len = self.blocks[edge.target.0].params.len(); + let args_len = edge.args.len(); + if target_len != args_len { + return Err(ValidationError::MismatchedBlockArity(block_id, target_len, args_len)); + } + Ok(()) + }; + + for block_id in self.reverse_post_order() { let insns = &self.blocks[block_id.0].insns; for (idx, insn_id) in insns.iter().enumerate() { let insn = self.find(*insn_id); + // Validate arity for all branch edges match &insn { - Insn::Jump(BranchEdge{target, args}) - | Insn::IfTrue { val: _, target: BranchEdge{target, args} } - | Insn::IfFalse { val: _, target: BranchEdge{target, args}} => { - let target_block = &self.blocks[target.0]; - let target_len = target_block.params.len(); - let args_len = args.len(); - if target_len != args_len { - return Err(ValidationError::MismatchedBlockArity(block_id, target_len, args_len)) - } + Insn::Jump(edge) => { + check_edge(block_id, edge)?; + } + Insn::CondBranch { if_true, if_false, .. } => { + check_edge(block_id, if_true)?; + check_edge(block_id, if_false)?; } _ => {} } - if !insn.is_terminator() { - continue; + + if insn.is_terminator() { + // Blow up if we have a terminator that isn't at the end + // of the block. + if idx != insns.len() - 1 { + return Err(ValidationError::TerminatorNotAtEnd(block_id, *insn_id, idx)) + } } - block_has_terminator = true; - if idx != insns.len() - 1 { - return Err(ValidationError::TerminatorNotAtEnd(block_id, *insn_id, idx)); + // If the last instruction isn't a terminator, return an error + if idx == insns.len() - 1 { + if !insn.is_terminator() { + return Err(ValidationError::BlockHasNoTerminator(block_id)); + } } } - if !block_has_terminator { - return Err(ValidationError::BlockHasNoTerminator(block_id)); - } } Ok(()) } @@ -2291,19 +5915,20 @@ impl Function { // Initialize with all missing values at first, to catch if a jump target points to a // missing location. let mut assigned_in = vec![None; self.num_blocks()]; - let rpo = self.rpo(); - // Begin with every block having every variable defined, except for the entry block, which + let rpo = self.reverse_post_order(); + // Begin with every block having every variable defined, except for entries_block, which // starts with nothing defined. - assigned_in[self.entry_block.0] = Some(InsnSet::with_capacity(self.insns.len())); for &block in &rpo { - if block != self.entry_block { + if block == self.entries_block { + assigned_in[block.0] = Some(InsnSet::with_capacity(self.insns.len())); + } else { let mut all_ones = InsnSet::with_capacity(self.insns.len()); all_ones.insert_all(); assigned_in[block.0] = Some(all_ones); } } let mut worklist = VecDeque::with_capacity(self.num_blocks()); - worklist.push_back(self.entry_block); + worklist.push_back(self.entries_block); while let Some(block) = worklist.pop_front() { let mut assigned = assigned_in[block.0].clone().unwrap(); for ¶m in &self.blocks[block.0].params { @@ -2311,14 +5936,25 @@ impl Function { } for &insn_id in &self.blocks[block.0].insns { let insn_id = self.union_find.borrow().find_const(insn_id); - match self.find(insn_id) { - Insn::Jump(target) | Insn::IfTrue { target, .. } | Insn::IfFalse { target, .. } => { - let Some(block_in) = assigned_in[target.target.0].as_mut() else { - return Err(ValidationError::JumpTargetNotInRPO(target.target)); - }; - // jump target's block_in was modified, we need to queue the block for processing. - if block_in.intersect_with(&assigned) { - worklist.push_back(target.target); + let insn = self.find(insn_id); + let mut propagate = |target: BlockId| -> Result<(), ValidationError> { + let Some(block_in) = assigned_in[target.0].as_mut() else { + return Err(ValidationError::JumpTargetNotInRPO(target)); + }; + if block_in.intersect_with(&assigned) { + worklist.push_back(target); + } + Ok(()) + }; + match insn { + Insn::Jump(edge) => propagate(edge.target)?, + Insn::CondBranch { if_true, if_false, .. } => { + propagate(if_true.target)?; + propagate(if_false.target)?; + } + Insn::Entries { ref targets } => { + for &target in targets { + propagate(target)?; } } insn if insn.has_output() => { @@ -2336,15 +5972,14 @@ impl Function { } for &insn_id in &self.blocks[block.0].insns { let insn_id = self.union_find.borrow().find_const(insn_id); - let mut operands = VecDeque::new(); - let insn = self.find(insn_id); - self.worklist_traverse_single_insn(&insn, &mut operands); - for operand in operands { + self.insns[insn_id.0].try_for_each_operand(|operand| { + let operand = self.union_find.borrow().find_const(operand); if !assigned.get(operand) { return Err(ValidationError::OperandNotDefined(block, insn_id, operand)); } - } - if insn.has_output() { + Ok(()) + })?; + if self.insns[insn_id.0].has_output() { assigned.insert(insn_id); } } @@ -2355,7 +5990,7 @@ impl Function { /// Checks that each instruction('s representative) appears only once in the CFG. fn validate_insn_uniqueness(&self) -> Result<(), ValidationError> { let mut seen = InsnSet::with_capacity(self.insns.len()); - for block_id in self.rpo() { + for block_id in self.reverse_post_order() { for &insn_id in &self.blocks[block_id.0].insns { let insn_id = self.union_find.borrow().find_const(insn_id); if !seen.insert(insn_id) { @@ -2366,11 +6001,344 @@ impl Function { Ok(()) } + fn assert_subtype(&self, user: InsnId, operand: InsnId, expected: Type) -> Result<(), ValidationError> { + let actual = self.type_of(operand); + if !actual.is_subtype(expected) { + return Err(ValidationError::MismatchedOperandType(user, operand, format!("{expected}"), format!("{actual}"))); + } + Ok(()) + } + + fn validate_insn_type(&self, insn_id: InsnId) -> Result<(), ValidationError> { + let insn_id = self.union_find.borrow().find_const(insn_id); + let insn = self.find(insn_id); + match insn { + // Instructions with no InsnId operands (except state) or nothing to assert + Insn::Const { .. } + | Insn::Comment { .. } + | Insn::Param + | Insn::LoadArg { .. } + | Insn::PutSpecialObject { .. } + | Insn::LoadField { .. } + | Insn::GetConstantPath { .. } + | Insn::IsBlockGiven { .. } + | Insn::GetGlobal { .. } + | Insn::LoadPC + | Insn::LoadSP + | Insn::LoadEC + | Insn::GetEP { .. } + | Insn::BreakPoint | Insn::Unreachable + | Insn::LoadSelf + | Insn::Snapshot { .. } + | Insn::Jump { .. } + | Insn::Entries { .. } + | Insn::EntryPoint { .. } + | Insn::PatchPoint { .. } + | Insn::SideExit { .. } + | Insn::IncrCounter { .. } + | Insn::IncrCounterPtr { .. } + | Insn::CheckInterrupts { .. } + | Insn::GetClassVar { .. } + | Insn::GetSpecialNumber { .. } + | Insn::GetSpecialSymbol { .. } + | Insn::GetBlockParam { .. } + | Insn::StoreField { .. } => { + Ok(()) + } + // Instructions with 1 Ruby object operand + Insn::Test { val } + | Insn::IsMethodCfunc { val, .. } + | Insn::SetGlobal { val, .. } + | Insn::SetLocal { val, .. } + | Insn::SetClassVar { val, .. } + | Insn::Return { val } + | Insn::Throw { val, .. } + | Insn::ObjToString { val, .. } + | Insn::GuardType { val, .. } + | Insn::ToArray { val, .. } + | Insn::ToNewArray { val, .. } + | Insn::Defined { v: val, .. } + | Insn::ObjectAlloc { val, .. } + | Insn::DupArrayInclude { target: val, .. } + | Insn::GetIvar { self_val: val, .. } + | Insn::CCall { recv: val, .. } + | Insn::FixnumBitCheck { val, .. } // TODO (https://github.com/Shopify/ruby/issues/859) this should check Fixnum, but then test_checkkeyword_tests_fixnum_bit fails + | Insn::DefinedIvar { self_val: val, .. } => { + self.assert_subtype(insn_id, val, types::BasicObject) + } + // Instructions with 2 Ruby object operands + Insn::SetIvar { self_val: left, val: right, .. } + | Insn::NewRange { low: left, high: right, .. } + | Insn::AnyToString { val: left, str: right, .. } + | Insn::CheckMatch { target: left, pattern: right, .. } + | Insn::WriteBarrier { recv: left, val: right } => { + self.assert_subtype(insn_id, left, types::BasicObject)?; + self.assert_subtype(insn_id, right, types::BasicObject) + } + Insn::GetConstant { klass, allow_nil, .. } => { + self.assert_subtype(insn_id, klass, types::BasicObject)?; + self.assert_subtype(insn_id, allow_nil, types::BoolExact) + } + // Instructions with recv and a Vec of Ruby objects + Insn::SendDirect { recv, ref args, .. } + | Insn::Send { recv, ref args, .. } + | Insn::SendForward { recv, ref args, .. } + | Insn::InvokeSuper { recv, ref args, .. } + | Insn::InvokeSuperForward { recv, ref args, .. } + | Insn::CCallWithFrame { recv, ref args, .. } + | Insn::CCallVariadic { recv, ref args, .. } + | Insn::InvokeBuiltin { recv, ref args, .. } + | Insn::InvokeProc { recv, ref args, .. } + | Insn::ArrayInclude { target: recv, elements: ref args, .. } => { + self.assert_subtype(insn_id, recv, types::BasicObject)?; + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::ArrayPackBuffer { ref elements, fmt, buffer, .. } => { + self.assert_subtype(insn_id, fmt, types::BasicObject)?; + if let Some(buffer) = buffer { + self.assert_subtype(insn_id, buffer, types::BasicObject)?; + } + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + // Instructions with a Vec of Ruby objects + Insn::InvokeBlock { ref args, .. } + | Insn::InvokeBlockIfunc { ref args, .. } + | Insn::NewArray { elements: ref args, .. } + | Insn::ArrayHash { elements: ref args, .. } + | Insn::ArrayMin { elements: ref args, .. } + | Insn::ArrayMax { elements: ref args, .. } => { + for &arg in args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::NewHash { ref elements, .. } => { + if elements.len() % 2 != 0 { + return Err(ValidationError::MiscValidationError(insn_id, "NewHash elements length is not even".to_string())); + } + for &element in elements { + self.assert_subtype(insn_id, element, types::BasicObject)?; + } + Ok(()) + } + Insn::StringConcat { ref strings, .. } + | Insn::ToRegexp { values: ref strings, .. } => { + for &string in strings { + self.assert_subtype(insn_id, string, types::String)?; + } + Ok(()) + } + // Instructions with String operands + Insn::StringCopy { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringIntern { val, .. } => self.assert_subtype(insn_id, val, types::StringExact), + Insn::StringAppend { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::String) + } + Insn::StringAppendCodepoint { recv, other, .. } => { + self.assert_subtype(insn_id, recv, types::StringExact)?; + self.assert_subtype(insn_id, other, types::Fixnum) + } + Insn::StringEqual { left, right } => { + self.assert_subtype(insn_id, left, types::String)?; + self.assert_subtype(insn_id, right, types::String) + } + // Instructions with Array operands + Insn::ArrayDup { val, .. } => self.assert_subtype(insn_id, val, types::ArrayExact), + Insn::ArrayExtend { left, right, .. } => { + // TODO(max): Do left and right need to be ArrayExact? + self.assert_subtype(insn_id, left, types::Array)?; + self.assert_subtype(insn_id, right, types::Array) + } + Insn::ArrayPush { array, .. } + | Insn::ArrayPop { array, .. } + | Insn::ArrayLength { array, .. } => { + self.assert_subtype(insn_id, array, types::Array) + } + Insn::ArrayAref { array, index } => { + self.assert_subtype(insn_id, array, types::Array)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + Insn::ArrayAset { array, index, .. } => { + self.assert_subtype(insn_id, array, types::ArrayExact)?; + self.assert_subtype(insn_id, index, types::CInt64) + } + Insn::AdjustBounds { index, length } => { + self.assert_subtype(insn_id, index, types::CInt64)?; + self.assert_subtype(insn_id, length, types::CInt64) + } + // Instructions with Hash operands + Insn::HashAref { hash, .. } + | Insn::HashAset { hash, .. } => self.assert_subtype(insn_id, hash, types::HashExact), + Insn::HashDup { val, .. } => self.assert_subtype(insn_id, val, types::HashExact), + // Other + Insn::ObjectAllocClass { class, .. } => { + if !class_has_leaf_allocator(class) { + return Err(ValidationError::MiscValidationError(insn_id, "ObjectAllocClass must have leaf allocator".to_string())); + } + Ok(()) + } + Insn::IsBitEqual { left, right } + | Insn::IsBitNotEqual { left, right } => { + if self.is_a(left, types::CInt) && self.is_a(right, types::CInt) { + // TODO(max): Check that int sizes match + Ok(()) + } else if self.is_a(left, types::CPtr) && self.is_a(right, types::CPtr) { + Ok(()) + } else if self.is_a(left, types::RubyValue) && self.is_a(right, types::RubyValue) { + Ok(()) + } else { + Err(ValidationError::MiscValidationError(insn_id, "IsBitEqual can only compare CInt/CInt or RubyValue/RubyValue".to_string())) + } + } + Insn::IntAnd { left, right } + | Insn::IntOr { left, right } => { + // TODO: Expand this to other matching C integer sizes when we need them. + let left_type = self.type_of(left); + if left_type.is_subtype(types::CInt64) { + self.assert_subtype(insn_id, right, types::CInt64) + } else if left_type.is_subtype(types::CUInt64) { + self.assert_subtype(insn_id, right, types::CUInt64) + } else { + let all_ints = types::CInt64.union(types::CUInt64); + self.assert_subtype(insn_id, left, all_ints)?; + self.assert_subtype(insn_id, right, all_ints) + } + } + Insn::BoxBool { val } + | Insn::CondBranch { val, .. } => { + self.assert_subtype(insn_id, val, types::CBool) + } + Insn::BoxFixnum { val, .. } => self.assert_subtype(insn_id, val, types::CInt64), + Insn::UnboxFixnum { val } => { + self.assert_subtype(insn_id, val, types::Fixnum) + } + Insn::FixnumAref { recv, index } => { + self.assert_subtype(insn_id, recv, types::Fixnum)?; + self.assert_subtype(insn_id, index, types::Fixnum) + } + Insn::FixnumAdd { left, right, .. } + | Insn::FixnumSub { left, right, .. } + | Insn::FixnumMult { left, right, .. } + | Insn::FixnumDiv { left, right, .. } + | Insn::FixnumMod { left, right, .. } + | Insn::FixnumEq { left, right } + | Insn::FixnumNeq { left, right } + | Insn::FixnumLt { left, right } + | Insn::FixnumLe { left, right } + | Insn::FixnumGt { left, right } + | Insn::FixnumGe { left, right } + | Insn::FixnumAnd { left, right } + | Insn::FixnumOr { left, right } + | Insn::FixnumXor { left, right } + | Insn::NewRangeFixnum { low: left, high: right, .. } + => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum) + } + Insn::FloatAdd { recv, other, .. } + | Insn::FloatSub { recv, other, .. } + | Insn::FloatMul { recv, other, .. } + | Insn::FloatDiv { recv, other, .. } + => { + self.assert_subtype(insn_id, recv, types::Flonum)?; + // other can be Flonum or Fixnum (rb_float_plus etc. handle both) + self.assert_subtype(insn_id, other, types::Flonum.union(types::Fixnum)) + } + Insn::FloatToInt { recv, .. } => { + self.assert_subtype(insn_id, recv, types::Flonum) + } + Insn::FixnumLShift { left, right, .. } + | Insn::FixnumRShift { left, right, .. } => { + self.assert_subtype(insn_id, left, types::Fixnum)?; + self.assert_subtype(insn_id, right, types::Fixnum)?; + let Some(obj) = self.type_of(right).fixnum_value() else { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<a compile-time constant>".into(), "<unknown>".into())); + }; + if obj < 0 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<positive>".into(), format!("{obj}"))); + } + if obj > 63 { + return Err(ValidationError::MismatchedOperandType(insn_id, right, "<less than 64>".into(), format!("{obj}"))); + } + Ok(()) + } + Insn::GuardBitEquals { val, expected, .. } => { + match expected { + Const::Value(_) => self.assert_subtype(insn_id, val, types::RubyValue), + Const::CInt8(_) => self.assert_subtype(insn_id, val, types::CInt8), + Const::CInt16(_) => self.assert_subtype(insn_id, val, types::CInt16), + Const::CInt32(_) => self.assert_subtype(insn_id, val, types::CInt32), + Const::CInt64(_) => self.assert_subtype(insn_id, val, types::CInt64), + Const::CUInt8(_) => self.assert_subtype(insn_id, val, types::CUInt8), + Const::CUInt16(_) => self.assert_subtype(insn_id, val, types::CUInt16), + Const::CUInt32(_) => self.assert_subtype(insn_id, val, types::CUInt32), + Const::CAttrIndex(_) => self.assert_subtype(insn_id, val, types::CAttrIndex), + Const::CShape(_) => self.assert_subtype(insn_id, val, types::CShape), + Const::CUInt64(_) => self.assert_subtype(insn_id, val, types::CUInt64), + Const::CBool(_) => self.assert_subtype(insn_id, val, types::CBool), + Const::CDouble(_) => self.assert_subtype(insn_id, val, types::CDouble), + Const::CPtr(_) => self.assert_subtype(insn_id, val, types::CPtr), + } + } + Insn::GuardAnyBitSet { val, mask, .. } + | Insn::GuardNoBitsSet { val, mask, .. } => { + match mask { + Const::CUInt8(_) | Const::CUInt16(_) | Const::CUInt32(_) | Const::CUInt64(_) + if self.is_a(val, types::CInt) || self.is_a(val, types::RubyValue) => { + Ok(()) + } + _ => { + Err(ValidationError::MiscValidationError(insn_id, "GuardAnyBitSet/GuardNoBitsSet can only compare RubyValue/CUInt or CInt/CUInt".to_string())) + } + } + } + Insn::GuardLess { left, right, .. } + | Insn::GuardGreaterEq { left, right, .. } => { + self.assert_subtype(insn_id, left, types::CInt64)?; + self.assert_subtype(insn_id, right, types::CInt64) + }, + Insn::StringGetbyte { string, index } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::CInt64) + }, + Insn::StringSetbyteFixnum { string, index, value } => { + self.assert_subtype(insn_id, string, types::String)?; + self.assert_subtype(insn_id, index, types::Fixnum)?; + self.assert_subtype(insn_id, value, types::Fixnum) + } + Insn::IsA { val, class } => { + self.assert_subtype(insn_id, val, types::BasicObject)?; + self.assert_subtype(insn_id, class, types::Class) + } + Insn::RefineType { .. } => Ok(()), + Insn::HasType { val, .. } => self.assert_subtype(insn_id, val, types::BasicObject), + Insn::IsBlockParamModified { flags } => self.assert_subtype(insn_id, flags, types::CUInt64), + } + } + + /// Check that insn types match the expected types for each instruction. + fn validate_types(&self) -> Result<(), ValidationError> { + for block_id in self.reverse_post_order() { + for &insn_id in &self.blocks[block_id.0].insns { + self.validate_insn_type(insn_id)?; + } + } + Ok(()) + } + /// Run all validation passes we have. pub fn validate(&self) -> Result<(), ValidationError> { self.validate_block_terminators_and_jumps()?; self.validate_definite_assignment()?; self.validate_insn_uniqueness()?; + self.validate_types()?; Ok(()) } } @@ -2378,7 +6346,13 @@ impl Function { impl<'a> std::fmt::Display for FunctionPrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { let fun = &self.fun; - let iseq_name = iseq_get_location(fun.iseq, 0); + // In tests, there may not be an iseq to get location from. + let iseq_name = if fun.iseq.is_null() { + String::from("<manual>") + } else { + iseq_get_location(fun.iseq, 0) + }; + // In tests, strip the line number for builtin ISEQs to make tests stable across line changes let iseq_name = if cfg!(test) && iseq_name.contains("@<internal:") { iseq_name[..iseq_name.rfind(':').unwrap()].to_string() @@ -2386,7 +6360,12 @@ impl<'a> std::fmt::Display for FunctionPrinter<'a> { iseq_name }; writeln!(f, "fn {iseq_name}:")?; - for block_id in fun.rpo() { + for block_id in fun.reverse_post_order() { + if !self.display_snapshot_and_tp_patchpoints && block_id == fun.entries_block { + // Unless we're doing --zjit-dump-hir=all, skip the entries superblock -- it's an + // internal CFG artifact + continue; + } write!(f, "{block_id}(")?; if !fun.blocks[block_id.0].params.is_empty() { let mut sep = ""; @@ -2402,7 +6381,8 @@ impl<'a> std::fmt::Display for FunctionPrinter<'a> { writeln!(f, "):")?; for insn_id in &fun.blocks[block_id.0].insns { let insn = fun.find(*insn_id); - if !self.display_snapshot && matches!(insn, Insn::Snapshot {..}) { + if !self.display_snapshot_and_tp_patchpoints && + matches!(insn, Insn::Snapshot {..} | Insn::PatchPoint { invariant: Invariant::NoTracePoint, .. }) { continue; } write!(f, " ")?; @@ -2414,105 +6394,68 @@ impl<'a> std::fmt::Display for FunctionPrinter<'a> { write!(f, "{insn_id}:{} = ", insn_type.print(&self.ptr_map))?; } } - writeln!(f, "{}", insn.print(&self.ptr_map))?; + writeln!(f, "{}", insn.print(&self.ptr_map, Some(fun.iseq)))?; } } Ok(()) } } -struct HtmlEncoder<'a, 'b> { - formatter: &'a mut std::fmt::Formatter<'b>, +#[derive(Debug, Clone, PartialEq)] +pub struct FrameState { + pub iseq: IseqPtr, + insn_idx: YarvInsnIdx, + // Ruby bytecode instruction pointer + pub pc: *const VALUE, + + stack: Vec<InsnId>, + locals: Vec<InsnId>, } -impl<'a, 'b> std::fmt::Write for HtmlEncoder<'a, 'b> { - fn write_str(&mut self, s: &str) -> std::fmt::Result { - for ch in s.chars() { - match ch { - '<' => self.formatter.write_str("<")?, - '>' => self.formatter.write_str(">")?, - '&' => self.formatter.write_str("&")?, - '"' => self.formatter.write_str(""")?, - '\'' => self.formatter.write_str("'")?, - _ => self.formatter.write_char(ch)?, - } - } - Ok(()) +impl FrameState { + /// Get the YARV instruction index for the current instruction + pub fn insn_idx(&self) -> YarvInsnIdx { + self.insn_idx } -} -impl<'a> std::fmt::Display for FunctionGraphvizPrinter<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - macro_rules! write_encoded { - ($f:ident, $($arg:tt)*) => { - HtmlEncoder { formatter: $f }.write_fmt(format_args!($($arg)*)) - }; - } - use std::fmt::Write; - let fun = &self.fun; - let iseq_name = iseq_get_location(fun.iseq, 0); - write!(f, "digraph G {{ # ")?; - write_encoded!(f, "{iseq_name}")?; - write!(f, "\n")?; - writeln!(f, "node [shape=plaintext];")?; - writeln!(f, "mode=hier; overlap=false; splines=true;")?; - for block_id in fun.rpo() { - writeln!(f, r#" {block_id} [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0">"#)?; - write!(f, r#"<TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">{block_id}("#)?; - if !fun.blocks[block_id.0].params.is_empty() { - let mut sep = ""; - for param in &fun.blocks[block_id.0].params { - write_encoded!(f, "{sep}{param}")?; - let insn_type = fun.type_of(*param); - if !insn_type.is_subtype(types::Empty) { - write_encoded!(f, ":{}", insn_type.print(&self.ptr_map))?; - } - sep = ", "; - } - } - let mut edges = vec![]; - writeln!(f, ") </TD></TR>")?; - for insn_id in &fun.blocks[block_id.0].insns { - let insn_id = fun.union_find.borrow().find_const(*insn_id); - let insn = fun.find(insn_id); - if matches!(insn, Insn::Snapshot {..}) { - continue; - } - write!(f, r#"<TR><TD ALIGN="left" PORT="{insn_id}">"#)?; - if insn.has_output() { - let insn_type = fun.type_of(insn_id); - if insn_type.is_subtype(types::Empty) { - write_encoded!(f, "{insn_id} = ")?; - } else { - write_encoded!(f, "{insn_id}:{} = ", insn_type.print(&self.ptr_map))?; - } - } - if let Insn::Jump(ref target) | Insn::IfTrue { ref target, .. } | Insn::IfFalse { ref target, .. } = insn { - edges.push((insn_id, target.target)); - } - write_encoded!(f, "{}", insn.print(&self.ptr_map))?; - writeln!(f, " </TD></TR>")?; + /// Return itself without locals. Useful for side-exiting without spilling locals. + fn without_locals(&self) -> Self { + let mut state = self.clone(); + state.locals.clear(); + state + } + + /// Return itself without stack. Used by leaf calls with GC to reset SP to the base pointer. + pub fn without_stack(&self) -> Self { + let mut state = self.clone(); + state.stack.clear(); + state + } + + /// Return itself with send args replaced. Used when kwargs are reordered/synthesized for callee. + /// `original_argc` is the number of args originally on the stack (before processing). + fn with_replaced_args(&self, new_args: &[InsnId], original_argc: usize) -> Self { + let mut state = self.clone(); + let args_start = state.stack.len() - original_argc; + state.stack.truncate(args_start); + state.stack.extend_from_slice(new_args); + state + } + + fn replace(&mut self, old: InsnId, new: InsnId) { + for slot in &mut self.stack { + if *slot == old { + *slot = new; } - writeln!(f, "</TABLE>>];")?; - for (src, dst) in edges { - writeln!(f, " {block_id}:{src} -> {dst}:params;")?; + } + for slot in &mut self.locals { + if *slot == old { + *slot = new; } } - writeln!(f, "}}") } } -#[derive(Debug, Clone, PartialEq)] -pub struct FrameState { - iseq: IseqPtr, - insn_idx: usize, - // Ruby bytecode instruction pointer - pub pc: *const VALUE, - - stack: Vec<InsnId>, - locals: Vec<InsnId>, -} - /// Print adaptor for [`FrameState`]. See [`PtrPrintMap`]. pub struct FrameStatePrinter<'a> { inner: &'a FrameState, @@ -2620,7 +6563,7 @@ impl FrameState { // TODO: Modify the register allocator to allow reusing an argument // of another basic block. let mut args = vec![self_param]; - args.extend(self.locals.iter().chain(self.stack.iter()).map(|op| *op)); + args.extend(self.locals.iter().chain(self.stack.iter()).copied()); args } @@ -2639,9 +6582,14 @@ impl Display for FrameStatePrinter<'_> { let inner = self.inner; write!(f, "FrameState {{ pc: {:?}, stack: ", self.ptr_map.map_ptr(inner.pc))?; write_vec(f, &inner.stack)?; - write!(f, ", locals: ")?; - write_vec(f, &inner.locals)?; - write!(f, " }}") + write!(f, ", locals: [")?; + for (idx, local) in inner.locals.iter().enumerate() { + let name: ID = unsafe { rb_zjit_local_id(inner.iseq, idx.try_into().unwrap()) }; + let name = name.contents_lossy(); + if idx > 0 { write!(f, ", ")?; } + write!(f, "{name}={local}")?; + } + write!(f, "] }}") } } @@ -2655,10 +6603,14 @@ fn insn_idx_at_offset(idx: u32, offset: i64) -> u32 { ((idx as isize) + (offset as isize)) as u32 } -fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec<u32> { +struct BytecodeInfo { + jump_targets: Vec<u32>, +} + +fn compute_bytecode_info(iseq: *const rb_iseq_t, opt_table: &[u32]) -> BytecodeInfo { let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; let mut insn_idx = 0; - let mut jump_targets = HashSet::new(); + let mut jump_targets: HashSet<u32> = opt_table.iter().copied().collect(); while insn_idx < iseq_size { // Get the current pc and opcode let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; @@ -2669,7 +6621,8 @@ fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec<u32> { .unwrap(); insn_idx += insn_len(opcode as usize); match opcode { - YARVINSN_branchunless | YARVINSN_jump | YARVINSN_branchif | YARVINSN_branchnil => { + YARVINSN_branchunless | YARVINSN_jump | YARVINSN_branchif | YARVINSN_branchnil + | YARVINSN_branchunless_without_ints | YARVINSN_jump_without_ints | YARVINSN_branchif_without_ints | YARVINSN_branchnil_without_ints => { let offset = get_arg(pc, 0).as_i64(); jump_targets.insert(insn_idx_at_offset(insn_idx, offset)); } @@ -2687,60 +6640,48 @@ fn compute_jump_targets(iseq: *const rb_iseq_t) -> Vec<u32> { } let mut result = jump_targets.into_iter().collect::<Vec<_>>(); result.sort(); - result + BytecodeInfo { jump_targets: result } } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum CallType { Splat, - BlockArg, Kwarg, - KwSplat, Tailcall, - Super, - Zsuper, - OptSend, - KwSplatMut, - SplatMut, - Forwarding, -} - -#[derive(Debug, PartialEq)] -pub enum ParameterType { - Optional, - /// For example, `foo(...)`. Interaction of JIT - /// calling convention and side exits currently unsolved. - Forwardable, } -#[derive(Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ParseError { StackUnderflow(FrameState), - UnknownParameterType(ParameterType), MalformedIseq(u32), // insn_idx into iseq_encoded Validation(ValidationError), NotAllowed, + DirectiveInduced, } /// Return the number of locals in the current ISEQ (includes parameters) fn num_locals(iseq: *const rb_iseq_t) -> usize { - (unsafe { get_iseq_body_local_table_size(iseq) }).as_usize() + (unsafe { get_iseq_body_local_table_size(iseq) }).to_usize() } /// If we can't handle the type of send (yet), bail out. -fn unknown_call_type(flag: u32) -> bool { - if (flag & VM_CALL_KW_SPLAT_MUT) != 0 { return true; } - if (flag & VM_CALL_ARGS_SPLAT_MUT) != 0 { return true; } - if (flag & VM_CALL_ARGS_SPLAT) != 0 { return true; } - if (flag & VM_CALL_KW_SPLAT) != 0 { return true; } - if (flag & VM_CALL_ARGS_BLOCKARG) != 0 { return true; } - if (flag & VM_CALL_KWARG) != 0 { return true; } - if (flag & VM_CALL_TAILCALL) != 0 { return true; } - if (flag & VM_CALL_SUPER) != 0 { return true; } - if (flag & VM_CALL_ZSUPER) != 0 { return true; } - if (flag & VM_CALL_OPT_SEND) != 0 { return true; } - if (flag & VM_CALL_FORWARDING) != 0 { return true; } - false +fn unhandled_call_type(flags: u32) -> Result<(), CallType> { + if (flags & VM_CALL_TAILCALL) != 0 { return Err(CallType::Tailcall); } + Ok(()) +} + +/// If a given call to a c func uses overly complex arguments, then we won't specialize. +fn unspecializable_c_call_type(flags: u32) -> bool { + ((flags & VM_CALL_KWARG) != 0) || + unspecializable_call_type(flags) +} + +/// If a given call uses overly complex arguments, then we won't specialize. +fn unspecializable_call_type(flags: u32) -> bool { + ((flags & VM_CALL_ARGS_SPLAT) != 0) || + ((flags & VM_CALL_KW_SPLAT) != 0) || + ((flags & VM_CALL_ARGS_BLOCKARG) != 0) || + ((flags & VM_CALL_FORWARDING) != 0) } /// We have IseqPayload, which keeps track of HIR Types in the interpreter, but this is not useful @@ -2753,7 +6694,7 @@ struct ProfileOracle { /// instruction index. At a given ISEQ instruction, the interpreter has profiled the stack /// operands to a given ISEQ instruction, and this list of pairs of (InsnId, Type) map that /// profiling information into HIR instructions. - types: HashMap<usize, Vec<(InsnId, TypeDistributionSummary)>>, + types: HashMap<YarvInsnIdx, Vec<(InsnId, TypeDistributionSummary)>>, } impl ProfileOracle { @@ -2761,11 +6702,11 @@ impl ProfileOracle { Self { payload, types: Default::default() } } - /// Map the interpreter-recorded types of the stack onto the HIR operands on our compile-time virtual stack + /// Map the interpreter-recorded types of the stack onto the HIR operands on our compile-time virtual stack. fn profile_stack(&mut self, state: &FrameState) { let iseq_insn_idx = state.insn_idx; let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; - let entry = self.types.entry(iseq_insn_idx).or_insert_with(|| vec![]); + let entry = self.types.entry(iseq_insn_idx).or_default(); // operand_types is always going to be <= stack size (otherwise it would have an underflow // at run-time) so use that to drive iteration. for (idx, insn_type_distribution) in operand_types.iter().rev().enumerate() { @@ -2773,93 +6714,121 @@ impl ProfileOracle { entry.push((insn, TypeDistributionSummary::new(insn_type_distribution))) } } + + /// Map the interpreter-recorded types of self onto the HIR self + fn profile_self(&mut self, state: &FrameState, self_param: InsnId) { + let iseq_insn_idx = state.insn_idx; + let Some(operand_types) = self.payload.profile.get_operand_types(iseq_insn_idx) else { return }; + let entry = self.types.entry(iseq_insn_idx).or_default(); + if operand_types.is_empty() { + return; + } + let self_type_distribution = &operand_types[0]; + entry.push((self_param, TypeDistributionSummary::new(self_type_distribution))) + } +} + +fn invalidates_locals(opcode: u32, operands: *const VALUE) -> bool { + match opcode { + // Control-flow is non-leaf in the interpreter because it can execute arbitrary code on + // interrupt. But in the JIT, we side-exit if there is a pending interrupt. + YARVINSN_jump + | YARVINSN_branchunless + | YARVINSN_branchif + | YARVINSN_branchnil + | YARVINSN_jump_without_ints + | YARVINSN_branchunless_without_ints + | YARVINSN_branchif_without_ints + | YARVINSN_branchnil_without_ints + | YARVINSN_leave => false, + // TODO(max): Read the invokebuiltin target from operands and determine if it's leaf + _ => unsafe { !rb_zjit_insn_leaf(opcode as i32, operands) } + } } /// The index of the self parameter in the HIR function pub const SELF_PARAM_IDX: usize = 0; -fn filter_unknown_parameter_type(iseq: *const rb_iseq_t) -> Result<(), ParseError> { - if unsafe { rb_get_iseq_body_param_opt_num(iseq) } != 0 { return Err(ParseError::UnknownParameterType(ParameterType::Optional)); } - if unsafe { rb_get_iseq_flags_forwardable(iseq) } { return Err(ParseError::UnknownParameterType(ParameterType::Forwardable)); } - Ok(()) -} - /// Compile ISEQ into High-level IR pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { if !ZJITState::can_compile_iseq(iseq) { return Err(ParseError::NotAllowed); } - filter_unknown_parameter_type(iseq)?; let payload = get_or_create_iseq_payload(iseq); let mut profiles = ProfileOracle::new(payload); let mut fun = Function::new(iseq); + fun.was_invalidated_for_singleton_class_creation = payload.was_invalidated_for_singleton_class_creation; + fun.self_is_heap_object = payload.self_is_heap_object; + // Compute a map of PC->Block by finding jump targets - let jump_targets = compute_jump_targets(iseq); + let jit_entry_insns = unsafe { iseq.params() }.opt_table_slice().iter().copied().map(VALUE::as_u32).collect::<Vec<_>>(); + let BytecodeInfo { jump_targets } = compute_bytecode_info(iseq, &jit_entry_insns); + + // Make all empty basic blocks. The ordering of the BBs matters for getting fallthrough jumps + // in good places, but it's not necessary for correctness. TODO: Higher quality scheduling during lowering. let mut insn_idx_to_block = HashMap::new(); + // Make blocks for optionals first, and put them right next to their JIT entrypoint + for insn_idx in jit_entry_insns.iter().copied() { + let jit_entry_block = fun.new_block(insn_idx); + fun.jit_entry_blocks.push(jit_entry_block); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); + } + // Make blocks for the rest of the jump targets for insn_idx in jump_targets { - if insn_idx == 0 { - todo!("Separate entry block for param/self/..."); - } - insn_idx_to_block.insert(insn_idx, fun.new_block(insn_idx)); + insn_idx_to_block.entry(insn_idx).or_insert_with(|| fun.new_block(insn_idx)); } + // Done, drop `mut`. + let insn_idx_to_block = insn_idx_to_block; - // Iteratively fill out basic blocks using a queue - // TODO(max): Basic block arguments at edges - let mut queue = std::collections::VecDeque::new(); - // Index of the rest parameter for comparison below - let rest_param_idx = if !iseq.is_null() && unsafe { get_iseq_flags_has_rest(iseq) } { - let opt_num = unsafe { get_iseq_body_param_opt_num(iseq) }; - let lead_num = unsafe { get_iseq_body_param_lead_num(iseq) }; - opt_num + lead_num - } else { - -1 - }; - // The HIR function will have the same number of parameter as the iseq so - // we properly handle calls from the interpreter. Roughly speaking, each - // item between commas in the source increase the parameter count by one, - // regardless of parameter kind. - let mut entry_state = FrameState::new(iseq); - fun.push_insn(fun.entry_block, Insn::Param { idx: SELF_PARAM_IDX }); - fun.param_types.push(types::BasicObject); // self - for local_idx in 0..num_locals(iseq) { - if local_idx < unsafe { get_iseq_body_param_size(iseq) }.as_usize() { - entry_state.locals.push(fun.push_insn(fun.entry_block, Insn::Param { idx: local_idx + 1 })); // +1 for self - } else { - entry_state.locals.push(fun.push_insn(fun.entry_block, Insn::Const { val: Const::Value(Qnil) })); - } + // Compile an entry_block for the interpreter + compile_entry_block(&mut fun, jit_entry_insns.as_slice(), &insn_idx_to_block); - let mut param_type = types::BasicObject; - // Rest parameters are always ArrayExact - if let Ok(true) = c_int::try_from(local_idx).map(|idx| idx == rest_param_idx) { - param_type = types::ArrayExact; - } - fun.param_types.push(param_type); + // Compile all JIT-to-JIT entry blocks + for (jit_entry_idx, insn_idx) in jit_entry_insns.iter().enumerate() { + let target_block = insn_idx_to_block.get(insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + compile_jit_entry_block(&mut fun, jit_entry_idx, target_block); } - queue.push_back((entry_state, fun.entry_block, /*insn_idx=*/0_u32)); - let mut visited = HashSet::new(); + // Check if the EP is escaped for the ISEQ from the beginning. We give up + // optimizing locals in that case because they're shared with other frames. + let ep_starts_escaped = iseq_escapes_ep(iseq); + // Check if the EP has been escaped at some point in the ISEQ. If it has, then we assume that + // its EP is shared with other frames. + let ep_has_been_escaped = crate::invariants::iseq_escapes_ep(iseq); + let ep_escaped = ep_starts_escaped || ep_has_been_escaped; + // Iteratively fill out basic blocks using a queue. + // TODO(max): Basic block arguments at edges + let mut queue = VecDeque::new(); + for &insn_idx in jit_entry_insns.iter() { + queue.push_back((FrameState::new(iseq), insn_idx_to_block[&insn_idx], /*insn_idx=*/insn_idx, /*local_inval=*/false)); + } + + // Keep compiling blocks until the queue becomes empty + let mut visited = HashSet::new(); let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; - let iseq_type = unsafe { get_iseq_body_type(iseq) }; - while let Some((incoming_state, block, mut insn_idx)) = queue.pop_front() { + while let Some((incoming_state, mut block, mut insn_idx, mut local_inval)) = queue.pop_front() { + // Compile each block only once if visited.contains(&block) { continue; } visited.insert(block); - let (self_param, mut state) = if insn_idx == 0 { - (fun.blocks[fun.entry_block.0].params[SELF_PARAM_IDX], incoming_state.clone()) - } else { - let self_param = fun.push_insn(block, Insn::Param { idx: SELF_PARAM_IDX }); + + // Load basic block params first + let mut self_param = fun.push_insn(block, Insn::Param); + let mut state = { let mut result = FrameState::new(iseq); - let mut idx = 1; - for _ in 0..incoming_state.locals.len() { - result.locals.push(fun.push_insn(block, Insn::Param { idx })); - idx += 1; + let local_size = if jit_entry_insns.contains(&insn_idx) { num_locals(iseq) } else { incoming_state.locals.len() }; + for _ in 0..local_size { + result.locals.push(fun.push_insn(block, Insn::Param)); } for _ in incoming_state.stack { - result.stack.push(fun.push_insn(block, Insn::Param { idx })); - idx += 1; + result.stack.push(fun.push_insn(block, Insn::Param)); } - (self_param, result) + result }; + // Start the block off with a Snapshot so that if we need to insert a new Guard later on // and we don't have a Snapshot handy, we can just iterate backward (at the earliest, to // the beginning of the block). @@ -2870,17 +6839,103 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; state.pc = pc; let exit_state = state.clone(); - profiles.profile_stack(&exit_state); - - // Increment zjit_insns_count for each YARV instruction if --zjit-stats is enabled. - if get_option!(stats) { - fun.push_insn(block, Insn::IncrCounter(Counter::zjit_insns_count)); - } // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } .try_into() .unwrap(); + + // If TracePoint has been enabled after we have collected profiles, we'll see + // trace_getinstancevariable in the ISEQ. We have to treat it like getinstancevariable + // for profiling purposes: there is no operand on the stack to look up; we have + // profiled cfp->self. + if opcode == YARVINSN_getinstancevariable || opcode == YARVINSN_trace_getinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_setinstancevariable || opcode == YARVINSN_trace_setinstancevariable { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_definedivar || opcode == YARVINSN_trace_definedivar { + profiles.profile_self(&exit_state, self_param); + } else if opcode == YARVINSN_invokeblock || opcode == YARVINSN_trace_invokeblock { + if get_option!(stats) { + let iseq_insn_idx = exit_state.insn_idx; + if let Some(operand_types) = profiles.payload.profile.get_operand_types(iseq_insn_idx) { + if let [self_type_distribution] = &operand_types[..] { + let summary = TypeDistributionSummary::new(&self_type_distribution); + if summary.is_monomorphic() { + let obj = summary.bucket(0).class(); + if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 } { + fun.count(block, Counter::invokeblock_handler_monomorphic_iseq); + } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 } { + fun.count(block, Counter::invokeblock_handler_monomorphic_ifunc); + } else { + fun.count(block, Counter::invokeblock_handler_monomorphic_other); + } + } else if summary.is_skewed_polymorphic() || summary.is_polymorphic() { + fun.count(block, Counter::invokeblock_handler_polymorphic); + } else if summary.is_skewed_megamorphic() || summary.is_megamorphic() { + fun.count(block, Counter::invokeblock_handler_megamorphic); + } else { + fun.count(block, Counter::invokeblock_handler_no_profiles); + } + } else { + fun.count(block, Counter::invokeblock_handler_no_profiles); + } + } + } + } else if opcode == YARVINSN_getblockparamproxy || opcode == YARVINSN_trace_getblockparamproxy { + if get_option!(stats) { + let iseq_insn_idx = exit_state.insn_idx; + if let Some([block_handler_distribution]) = profiles.payload.profile.get_operand_types(iseq_insn_idx) { + let summary = TypeDistributionSummary::new(block_handler_distribution); + + if summary.is_monomorphic() { + let obj = summary.bucket(0).class(); + if unsafe { rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1} { + fun.count(block, Counter::getblockparamproxy_handler_iseq); + } else if unsafe { rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1} { + fun.count(block, Counter::getblockparamproxy_handler_ifunc); + } + else if obj.nil_p() { + fun.count(block, Counter::getblockparamproxy_handler_nil); + } + else if obj.symbol_p() { + fun.count(block, Counter::getblockparamproxy_handler_symbol); + } else if unsafe { rb_obj_is_proc(obj).test() } { + fun.count(block, Counter::getblockparamproxy_handler_proc); + } + } else if summary.is_polymorphic() || summary.is_skewed_polymorphic() { + fun.count(block, Counter::getblockparamproxy_handler_polymorphic); + } else if summary.is_megamorphic() || summary.is_skewed_megamorphic() { + fun.count(block, Counter::getblockparamproxy_handler_megamorphic); + } + } else { + fun.count(block, Counter::getblockparamproxy_handler_no_profiles); + } + } + } + else { + profiles.profile_stack(&exit_state); + } + + // Flag a future getlocal/setlocal to add a patch point if this instruction is not leaf. + if invalidates_locals(opcode, unsafe { pc.offset(1) }) { + local_inval = true; + } + + // We add NoTracePoint patch points before every instruction that could be affected by TracePoint. + // This ensures that if TracePoint is enabled, we can exit the generated code as fast as possible. + unsafe extern "C" { + fn rb_iseq_event_flags(iseq: IseqPtr, pos: usize) -> rb_event_flag_t; + } + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.clone() }); + if unsafe { rb_iseq_event_flags(iseq, insn_idx as usize) } != 0 { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoTracePoint, state: exit_id }); + } + + // Increment zjit_insn_count for each YARV instruction if --zjit-stats is enabled. + if get_option!(stats) { + fun.push_insn(block, Insn::IncrCounter(Counter::zjit_insn_count)); + } // Move to the next instruction to compile insn_idx += insn_len(opcode as usize); @@ -2893,32 +6948,28 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let insn = if value_type == SpecialObjectType::VMCore { Insn::Const { val: Const::Value(unsafe { rb_mRubyVMFrozenCore }) } } else { - Insn::PutSpecialObject { value_type } + Insn::PutSpecialObject { value_type, state: exit_id } }; state.stack_push(fun.push_insn(block, insn)); } - YARVINSN_putstring => { + YARVINSN_dupstring => { let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: false, state: exit_id }); state.stack_push(insn_id); } - YARVINSN_putchilledstring => { + YARVINSN_dupchilledstring => { let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let insn_id = fun.push_insn(block, Insn::StringCopy { val, chilled: true, state: exit_id }); state.stack_push(insn_id); } YARVINSN_putself => { state.stack_push(self_param); } YARVINSN_intern => { let val = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let insn_id = fun.push_insn(block, Insn::StringIntern { val, state: exit_id }); state.stack_push(insn_id); } YARVINSN_concatstrings => { let count = get_arg(pc, 0).as_u32(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let strings = state.stack_pop_n(count as usize)?; let insn_id = fun.push_insn(block, Insn::StringConcat { strings, state: exit_id }); state.stack_push(insn_id); @@ -2927,33 +6978,56 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { // First arg contains the options (multiline, extended, ignorecase) used to create the regexp let opt = get_arg(pc, 0).as_usize(); let count = get_arg(pc, 1).as_usize(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let values = state.stack_pop_n(count)?; let insn_id = fun.push_insn(block, Insn::ToRegexp { opt, values, state: exit_id }); state.stack_push(insn_id); } YARVINSN_newarray => { let count = get_arg(pc, 0).as_usize(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let elements = state.stack_pop_n(count)?; state.stack_push(fun.push_insn(block, Insn::NewArray { elements, state: exit_id })); } YARVINSN_opt_newarray_send => { let count = get_arg(pc, 0).as_usize(); let method = get_arg(pc, 1).as_u32(); - let elements = state.stack_pop_n(count)?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let (bop, insn) = match method { - VM_OPT_NEWARRAY_SEND_MAX => (BOP_MAX, Insn::ArrayMax { elements, state: exit_id }), + VM_OPT_NEWARRAY_SEND_MAX => { + let elements = state.stack_pop_n(count)?; + (BOP_MAX, Insn::ArrayMax { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_MIN => { + let elements = state.stack_pop_n(count)?; + (BOP_MIN, Insn::ArrayMin { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_HASH => { + let elements = state.stack_pop_n(count)?; + (BOP_HASH, Insn::ArrayHash { elements, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_INCLUDE_P => { + let target = state.stack_pop()?; + let elements = state.stack_pop_n(count - 1)?; + (BOP_INCLUDE_P, Insn::ArrayInclude { elements, target, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_PACK => { + let fmt = state.stack_pop()?; + let elements = state.stack_pop_n(count - 1)?; + (BOP_PACK, Insn::ArrayPackBuffer { elements, fmt, buffer: None, state: exit_id }) + } + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => { + let buffer = state.stack_pop()?; + let fmt = state.stack_pop()?; + let elements = state.stack_pop_n(count - 2)?; + (BOP_PACK, Insn::ArrayPackBuffer { elements, fmt, buffer: Some(buffer), state: exit_id }) + } _ => { // Unknown opcode; side-exit into the interpreter - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownNewarraySend(method) }); + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledNewarraySend(method), recompile: None }); break; // End the block - }, + } }; if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { // If the basic operation is already redefined, we cannot optimize it. - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }) }); + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }), recompile: None }); break; // End the block } fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); @@ -2961,26 +7035,47 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { } YARVINSN_duparray => { let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let insn_id = fun.push_insn(block, Insn::ArrayDup { val, state: exit_id }); state.stack_push(insn_id); } + YARVINSN_opt_duparray_send => { + let ary = get_arg(pc, 0); + let method_id = get_arg(pc, 1).as_u64(); + let argc = get_arg(pc, 2).as_usize(); + if argc != 1 { + break; + } + let target = state.stack_pop()?; + let bop = match method_id { + x if x == ID!(include_p).0 => BOP_INCLUDE_P, + _ => { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDuparraySend(method_id), recompile: None }); + break; + }, + }; + if !unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG) } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }), recompile: None }); + break; + } + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass: ARRAY_REDEFINED_OP_FLAG, bop }, state: exit_id }); + let insn_id = fun.push_insn(block, Insn::DupArrayInclude { ary, target, state: exit_id }); + state.stack_push(insn_id); + } YARVINSN_newhash => { let count = get_arg(pc, 0).as_usize(); assert!(count % 2 == 0, "newhash count should be even"); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let mut elements = vec![]; for _ in 0..(count/2) { let value = state.stack_pop()?; let key = state.stack_pop()?; - elements.push((key, value)); + elements.push(value); + elements.push(key); } elements.reverse(); state.stack_push(fun.push_insn(block, Insn::NewHash { elements, state: exit_id })); } YARVINSN_duphash => { let val = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let insn_id = fun.push_insn(block, Insn::HashDup { val, state: exit_id }); state.stack_push(insn_id); } @@ -2988,7 +7083,6 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let flag = get_arg(pc, 0); let result_must_be_mutable = flag.test(); let val = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let obj = if result_must_be_mutable { fun.push_insn(block, Insn::ToNewArray { val, state: exit_id }) } else { @@ -2996,10 +7090,36 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { }; state.stack_push(obj); } + YARVINSN_splatkw => { + let block_val = state.stack_pop()?; + let hash = state.stack_pop()?; + // Get profiled type of hash (operand index 0) + let summary = profiles.payload.profile.get_operand_types(exit_state.insn_idx) + .and_then(|types| types.first()) + .map(|dist| TypeDistributionSummary::new(dist)); + let Some(summary) = summary else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotProfiled, recompile: None }); + break; // End the block + }; + if !summary.is_monomorphic() { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwPolymorphic, recompile: None }); + break; // End the block + } + let ty = Type::from_profiled_type(summary.bucket(0)); + let obj = if ty.is_subtype(types::NilClass) { + fun.push_insn(block, Insn::GuardType { val: hash, guard_type: types::NilClass, state: exit_id, recompile: None }) + } else if ty.is_subtype(types::HashExact) { + fun.push_insn(block, Insn::GuardType { val: hash, guard_type: types::HashExact, state: exit_id, recompile: None }) + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SplatKwNotNilOrHash, recompile: None }); + break; // End the block + }; + state.stack_push(obj); + state.stack_push(block_val); + } YARVINSN_concattoarray => { let right = state.stack_pop()?; let left = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let right_array = fun.push_insn(block, Insn::ToArray { val: right, state: exit_id }); fun.push_insn(block, Insn::ArrayExtend { left, right: right_array, state: exit_id }); state.stack_push(left); @@ -3008,7 +7128,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let count = get_arg(pc, 0).as_usize(); let vals = state.stack_pop_n(count)?; let array = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); + fun.guard_not_frozen(block, array, exit_id); for val in vals.into_iter() { fun.push_insn(block, Insn::ArrayPush { array, val, state: exit_id }); } @@ -3026,120 +7146,345 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let obj = get_arg(pc, 1); let pushval = get_arg(pc, 2); let v = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - if op_type == DEFINED_METHOD.try_into().unwrap() { - // TODO(Shopify/ruby#703): Fix codegen for defined?(method call expr) - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledDefinedType(op_type)}); - break; // End the block - } - state.stack_push(fun.push_insn(block, Insn::Defined { op_type, obj, pushval, v, state: exit_id })); + let local_iseq = unsafe { rb_get_iseq_body_local_iseq(iseq) }; + let insn = if op_type == DEFINED_YIELD as usize && unsafe { rb_get_iseq_body_type(local_iseq) } != ISEQ_TYPE_METHOD { + // `yield` goes to the block handler stowed in the "local" iseq which is + // the current iseq or a parent. Only the "method" iseq type can be passed a + // block handler. (e.g. `yield` in the top level script is a syntax error.) + // + // Similar to gen_is_block_given + Insn::Const { val: Const::Value(Qnil) } + } else { + // For DEFINED_YIELD, codegen materializes the local EP inline (similar to + // gen_is_block_given) to check for a block handler. Precompute the lexical + // distance from this iseq up to local_iseq so codegen does not have to + // walk the parent chain. Any DEFINED_YIELD reaching this branch has a + // method local_iseq by construction -- the above branch has already + // diverted the non-method case to Qnil. + let lep_level = if op_type == DEFINED_YIELD as usize { + get_lvar_level(iseq) + } else { + 0 + }; + Insn::Defined { op_type, obj, pushval, v, lep_level, state: exit_id } + }; + state.stack_push(fun.push_insn(block, insn)); } YARVINSN_definedivar => { // (ID id, IVC ic, VALUE pushval) let id = ID(get_arg(pc, 0).as_u64()); let pushval = get_arg(pc, 2); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - state.stack_push(fun.push_insn(block, Insn::DefinedIvar { self_val: self_param, id, pushval, state: exit_id })); + if let Some(summary) = fun.polymorphic_summary(&profiles, self_param, exit_state.insn_idx) { + self_param = fun.push_insn(block, Insn::GuardType { val: self_param, guard_type: types::HeapBasicObject, state: exit_id, recompile: None }); + let rbasic_flags = fun.load_rbasic_flags(block, self_param); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + let join_param = fun.push_insn(join_block, Insn::Param); + // Dedup by expected shape and type so objects with different classes + // but the same shape can share code. + let mut seen_shape_and_flags = Vec::with_capacity(summary.buckets().len()); + for &profiled_type in summary.buckets() { + // End of the buckets + if profiled_type.is_empty() { break; } + // Runtime immediates cannot pass the HeapBasicObject guard, so don't + // generate unreachable shape branches for profiled immediate buckets. + if profiled_type.flags().is_immediate() { continue; } + // Class/module/T_DATA ivars use different storage rules. + // Let the fallthrough DefinedIvar handle these. + if !profiled_type.flags().is_t_object() { continue; } + let expected_shape = profiled_type.shape(); + let (expected_rbasic_flags, rbasic_flags_mask) = profiled_type.rbasic_flags_and_mask(); + assert!(expected_shape.is_valid()); + // Too-complex shapes use hash tables for ivars; + // rb_shape_get_iv_index doesn't work for them. + // Let the fallthrough DefinedIvar handle these. + if expected_shape.is_complex() { continue; } + if seen_shape_and_flags.contains(&expected_rbasic_flags) { continue; } + seen_shape_and_flags.push(expected_rbasic_flags); + let rbasic_flags_mask = fun.push_insn(block, Insn::Const { val: Const::CUInt64(rbasic_flags_mask) }); + // The expected shape can change over run, so we put it + // as a pointer to keep it stable in snapshot tests. + let expected_rbasic_flags = fun.push_insn(block, Insn::Const { val: Const::CPtr(ptr::without_provenance(expected_rbasic_flags.to_usize())) }); + let expected_rbasic_flags = fun.push_insn(block, Insn::RefineType { val: expected_rbasic_flags, new_type: types::CUInt64 }); + let masked = fun.push_insn(block, Insn::IntAnd { left: rbasic_flags, right: rbasic_flags_mask}); + let has_shape_and_type = fun.push_insn(block, Insn::IsBitEqual { left: masked, right: expected_rbasic_flags }); + let iftrue_block = fun.new_block(insn_idx); + let target = BranchEdge { target: iftrue_block, args: vec![] }; + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { val: has_shape_and_type, + if_true: target, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + + block = fall_through; + let mut ivar_index: attr_index_t = 0; + let result = if unsafe { rb_shape_get_iv_index(expected_shape.0, id, &mut ivar_index) } { + fun.push_insn(iftrue_block, Insn::Const { val: Const::Value(pushval) }) + } else { + fun.push_insn(iftrue_block, Insn::Const { val: Const::Value(Qnil) }) + }; + fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![result] })); + } + // In the fallthrough case, do a generic interpreter definedivar and then join. + let result = fun.push_insn(block, Insn::DefinedIvar { self_val: self_param, id, pushval, state: exit_id }); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![result] })); + state.stack_push(join_param); + block = join_block; + } else { + // TODO: Handle monomorphic definedivar specialization here too, including the + // no_side_exits policy, so optimize_getivar doesn't need a separate DefinedIvar + // path. Unlike GetIvar, DefinedIvar isn't emitted by later lowering passes. + state.stack_push(fun.push_insn(block, Insn::DefinedIvar { self_val: self_param, id, pushval, state: exit_id })); + } + } + YARVINSN_checkkeyword => { + // When a keyword is unspecified past index 32, a hash will be used instead. + // This can only happen in iseqs taking more than 32 keywords. + // In this case, we side exit to the interpreter. + if unsafe {(*rb_get_iseq_body_param_keyword(iseq)).num >= VM_KW_SPECIFIED_BITS_MAX.try_into().unwrap()} { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyKeywordParameters, recompile: None }); + break; + } + let ep_offset = get_arg(pc, 0).as_u32(); + let index = get_arg(pc, 1).as_u64(); + let index: u8 = index.try_into().map_err(|_| ParseError::MalformedIseq(insn_idx))?; + // Use FrameState to get kw_bits when possible, just like getlocal_WC_0. + let val = if !local_inval { + state.getlocal(ep_offset) + } else if ep_escaped { + let ep = fun.push_insn(block, Insn::GetEP { level: 0 }); + fun.get_local_from_ep(block, ep, ep_offset, 0, types::BasicObject) + } else { + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + state.getlocal(ep_offset) + }; + state.stack_push(fun.push_insn(block, Insn::FixnumBitCheck { val, index })); + } + YARVINSN_checkmatch => { + let flag = get_arg(pc, 0).as_u32(); + let pattern = state.stack_pop()?; + let target = state.stack_pop()?; + let result = fun.push_insn(block, Insn::CheckMatch { target, pattern, flag, state: exit_id }); + state.stack_push(result); + } + YARVINSN_getconstant => { + let id = ID(get_arg(pc, 0).as_u64()); + let allow_nil = state.stack_pop()?; + let klass = state.stack_pop()?; + let result = fun.push_insn(block, Insn::GetConstant { klass, id, allow_nil, state: exit_id }); + state.stack_push(result); } YARVINSN_opt_getconstant_path => { let ic = get_arg(pc, 0).as_ptr(); - let snapshot = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - state.stack_push(fun.push_insn(block, Insn::GetConstantPath { ic, state: snapshot })); + let get_const_path = fun.push_insn(block, Insn::GetConstantPath { ic, state: exit_id }); + state.stack_push(get_const_path); + + // Check for `::RubyVM::ZJIT` for directives + unsafe { + let mut current_segment = (*ic).segments; + let mut segments = [ID(0); 4 /* expected segment length */]; + for segment in segments.iter_mut() { + *segment = current_segment.read(); + if *segment == ID(0) { + break; + } + current_segment = current_segment.add(1); + } + if [ID!(NULL), ID!(RubyVM), ID!(ZJIT), ID(0)] == segments { + debug_assert_ne!(ID!(NULL), ID(0)); + let ruby_vm_mod = rb_const_lookup(rb_cObject, ID!(RubyVM)); + if !ruby_vm_mod.is_null() && (*ruby_vm_mod).value == rb_cRubyVM { + let zjit_module = VALUE(state::ZJIT_MODULE.load(Ordering::Relaxed)); + let lookedup_module = rb_const_lookup(rb_cRubyVM, ID!(ZJIT)); + if !lookedup_module.is_null() && (*lookedup_module).value == zjit_module { + fun.insn_types[get_const_path.0] = Type::from_value(zjit_module); + } + } + } + } } - YARVINSN_branchunless => { - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + YARVINSN_branchunless | YARVINSN_branchunless_without_ints => { + if opcode == YARVINSN_branchunless { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } let offset = get_arg(pc, 0).as_i64(); let val = state.stack_pop()?; let test_id = fun.push_insn(block, Insn::Test { val }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; - let _branch_id = fun.push_insn(block, Insn::IfFalse { + let nil_false_type = types::Falsy; + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + let mut iffalse_state = state.clone(); + iffalse_state.replace(val, nil_false); + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { val: test_id, - target: BranchEdge { target, args: state.as_args(self_param) } + if_true: BranchEdge { target: fall_through, args: vec![] }, + if_false: BranchEdge { target, args: iffalse_state.as_args(self_param) } }); - queue.push_back((state.clone(), target, target_idx)); + + block = fall_through; + + let not_nil_false_type = types::Truthy; + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + state.replace(val, not_nil_false); + queue.push_back((state.clone(), target, target_idx, local_inval)); } - YARVINSN_branchif => { - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + YARVINSN_branchif | YARVINSN_branchif_without_ints => { + if opcode == YARVINSN_branchif { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } let offset = get_arg(pc, 0).as_i64(); let val = state.stack_pop()?; let test_id = fun.push_insn(block, Insn::Test { val }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; - let _branch_id = fun.push_insn(block, Insn::IfTrue { + let not_nil_false_type = types::Truthy; + let not_nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: not_nil_false_type }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, not_nil_false); + + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { val: test_id, - target: BranchEdge { target, args: state.as_args(self_param) } + if_true: BranchEdge { target, args: iftrue_state.as_args(self_param) }, + if_false: BranchEdge { target: fall_through, args: vec![] } }); - queue.push_back((state.clone(), target, target_idx)); + + block = fall_through; + + let nil_false_type = types::Falsy; + let nil_false = fun.push_insn(block, Insn::RefineType { val, new_type: nil_false_type }); + state.replace(val, nil_false); + queue.push_back((state.clone(), target, target_idx, local_inval)); } - YARVINSN_branchnil => { - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + YARVINSN_branchnil | YARVINSN_branchnil_without_ints => { + if opcode == YARVINSN_branchnil { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } let offset = get_arg(pc, 0).as_i64(); let val = state.stack_pop()?; - let test_id = fun.push_insn(block, Insn::IsNil { val }); + let test_id = fun.push_insn(block, Insn::HasType { val, expected: types::NilClass }); let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; - let _branch_id = fun.push_insn(block, Insn::IfTrue { + let nil = fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let mut iftrue_state = state.clone(); + iftrue_state.replace(val, nil); + + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { val: test_id, - target: BranchEdge { target, args: state.as_args(self_param) } + if_true: BranchEdge { target, args: iftrue_state.as_args(self_param) }, + if_false: BranchEdge { target: fall_through, args: vec![] } }); - queue.push_back((state.clone(), target, target_idx)); + + block = fall_through; + let new_type = types::NotNil; + let not_nil = fun.push_insn(block, Insn::RefineType { val, new_type }); + state.replace(val, not_nil); + queue.push_back((state.clone(), target, target_idx, local_inval)); + } + YARVINSN_opt_case_dispatch => { + // TODO: Some keys are visible at compile time, so in the future we can + // compile jump targets for certain cases + // Pop the key from the stack and fallback to the === branches for now + state.stack_pop()?; } YARVINSN_opt_new => { - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); - let offset = get_arg(pc, 1).as_i64(); - let target_idx = insn_idx_at_offset(insn_idx, offset); + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let dst = get_arg(pc, 1).as_i64(); + + // Check if #new resolves to rb_class_new_instance_pass_kw. + // TODO: Guard on a profiled class and add a patch point for #new redefinition + let argc = crate::profile::num_arguments_on_stack(cd); + let ci = unsafe { get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + assert_eq!(flags & VM_CALL_ARGS_BLOCKARG, 0); + let val = state.stack_topn(argc)?; + let test_id = fun.push_insn(block, Insn::IsMethodCfunc { val, cd, cfunc: rb_class_new_instance_pass_kw as *const u8, state: exit_id }); + + // Jump to the fallback block if it's not the expected function. + // Skip CheckInterrupts since the #new call will do it very soon anyway. + let target_idx = insn_idx_at_offset(insn_idx, dst); let target = insn_idx_to_block[&target_idx]; - // Skip the fast-path and go straight to the fallback code. We will let the - // optimizer take care of the converting Class#new->alloc+initialize instead. - fun.push_insn(block, Insn::Jump(BranchEdge { target, args: state.as_args(self_param) })); - queue.push_back((state.clone(), target, target_idx)); - break; // Don't enqueue the next block as a successor + let fall_through = fun.new_block(insn_idx); + fun.push_insn(block, Insn::CondBranch { + val: test_id, + if_true: BranchEdge { target: fall_through, args: vec![] }, + if_false: BranchEdge { target, args: state.as_args(self_param) } + }); + block = fall_through; + queue.push_back((state.clone(), target, target_idx, local_inval)); + + // Move on to the fast path + let insn_id = fun.push_insn(block, Insn::ObjectAlloc { val, state: exit_id }); + state.stack_setn(argc, insn_id); + state.stack_setn(argc + 1, insn_id); } - YARVINSN_jump => { + YARVINSN_jump | YARVINSN_jump_without_ints => { let offset = get_arg(pc, 0).as_i64(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + if opcode == YARVINSN_jump { + fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); + } let target_idx = insn_idx_at_offset(insn_idx, offset); let target = insn_idx_to_block[&target_idx]; let _branch_id = fun.push_insn(block, Insn::Jump( BranchEdge { target, args: state.as_args(self_param) } )); - queue.push_back((state.clone(), target, target_idx)); + queue.push_back((state.clone(), target, target_idx, local_inval)); break; // Don't enqueue the next block as a successor } YARVINSN_getlocal_WC_0 => { let ep_offset = get_arg(pc, 0).as_u32(); - if iseq_type == ISEQ_TYPE_EVAL { - // On eval, the locals are always on the heap, so read the local using EP. - state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level: 0 })); + if !local_inval { + // The FrameState is the source of truth for locals until invalidated. + // In case of JIT-to-JIT send locals might never end up in EP memory. + let val = state.getlocal(ep_offset); + state.stack_push(val); + } else if ep_escaped { + // Read the local using EP + let ep = fun.push_insn(block, Insn::GetEP { level: 0 }); + let val = fun.get_local_from_ep(block, ep, ep_offset, 0, types::BasicObject); + state.setlocal(ep_offset, val); // remember the result to spill on side-exits + state.stack_push(val); } else { - // TODO(alan): This implementation doesn't read from EP, so will miss writes - // from nested ISeqs. This will need to be amended when we add codegen for - // Send. + assert!(local_inval); // if check above + // There has been some non-leaf call since JIT entry or the last patch point, + // so add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; + + // Read the local from FrameState let val = state.getlocal(ep_offset); state.stack_push(val); } } YARVINSN_setlocal_WC_0 => { - // TODO(alan): This implementation doesn't write to EP, where nested scopes - // read, so they'll miss these writes. This will need to be amended when we - // add codegen for Send. let ep_offset = get_arg(pc, 0).as_u32(); let val = state.stack_pop()?; - state.setlocal(ep_offset, val); - if iseq_type == ISEQ_TYPE_EVAL { - // On eval, the locals are always on the heap, so write the local using EP. + if ep_escaped { + // Write the local using EP fun.push_insn(block, Insn::SetLocal { val, ep_offset, level: 0 }); + } else if local_inval { + // If there has been any non-leaf call since JIT entry or the last patch point, + // add a patch point to make sure locals have not been escaped. + let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); + local_inval = false; } + // Write the local into FrameState + state.setlocal(ep_offset, val); } YARVINSN_getlocal_WC_1 => { let ep_offset = get_arg(pc, 0).as_u32(); - state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level: 1 })); + let ep = fun.push_insn(block, Insn::GetEP { level: 1 }); + state.stack_push(fun.get_local_from_ep(block, ep, ep_offset, 1, types::BasicObject)); } YARVINSN_setlocal_WC_1 => { let ep_offset = get_arg(pc, 0).as_u32(); @@ -3148,13 +7493,300 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { YARVINSN_getlocal => { let ep_offset = get_arg(pc, 0).as_u32(); let level = get_arg(pc, 1).as_u32(); - state.stack_push(fun.push_insn(block, Insn::GetLocal { ep_offset, level })); + if level == 0 && !local_inval { + // Same optimization as getlocal_WC_0: use FrameState + let val = state.getlocal(ep_offset); + state.stack_push(val); + } else { + let ep = fun.push_insn(block, Insn::GetEP { level }); + let val = fun.get_local_from_ep(block, ep, ep_offset, level, types::BasicObject); + if level == 0 { + state.setlocal(ep_offset, val); + } + state.stack_push(val); + } } YARVINSN_setlocal => { let ep_offset = get_arg(pc, 0).as_u32(); let level = get_arg(pc, 1).as_u32(); fun.push_insn(block, Insn::SetLocal { val: state.stack_pop()?, ep_offset, level }); } + YARVINSN_setblockparam => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetLocal { val, ep_offset, level }); + if level == 0 { + state.setlocal(ep_offset, val); + } + let ep = fun.push_insn(block, Insn::GetEP { level }); + let flags = fun.push_insn(block, Insn::LoadField { + recv: ep, + id: FieldName::VM_ENV_DATA_INDEX_FLAGS, + offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), + return_type: types::CInt64, + }); + let modified_flag = fun.push_insn(block, Insn::Const { + val: Const::CInt64(VM_FRAME_FLAG_MODIFIED_BLOCK_PARAM.into()), + }); + let modified = fun.push_insn(block, Insn::IntOr { left: flags, right: modified_flag }); + fun.push_insn(block, Insn::StoreField { + recv: ep, + id: FieldName::VM_ENV_DATA_INDEX_FLAGS, + offset: SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32), + val: modified, + }); + } + YARVINSN_getblockparamproxy => { + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] + enum ProfiledBlockHandlerFamily { + Nil, + IseqOrIfunc, + } + impl ProfiledBlockHandlerFamily { + fn from_profiled_type(profiled_type: ProfiledType) -> Option<Self> { + let obj = profiled_type.class(); + if obj.nil_p() { + Some(Self::Nil) + } else if unsafe { + rb_IMEMO_TYPE_P(obj, imemo_iseq) == 1 + || rb_IMEMO_TYPE_P(obj, imemo_ifunc) == 1 + } { + Some(Self::IseqOrIfunc) + } else { + None + } + } + } + + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + let branch_insn_idx = exit_state.insn_idx as u32; + + // `getblockparamproxy` has two semantic paths: + // - modified: return the already-materialized block local from EP + // - unmodified: inspect the block handler and produce proxy/nil + let modified_block = fun.new_block(branch_insn_idx); + let unmodified_block = fun.new_block(branch_insn_idx); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + let join_result = fun.push_insn(join_block, Insn::Param); + let join_local = if level == 0 { Some(fun.push_insn(join_block, Insn::Param)) } else { None }; + + let ep = fun.push_insn(block, Insn::GetEP { level }); + let flags = fun.load_ep_flags(block, ep); + let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { flags }); + + fun.push_insn(block, Insn::CondBranch { + val: is_modified, + if_true: BranchEdge { target: modified_block, args: vec![] }, + if_false: BranchEdge { target: unmodified_block, args: vec![] } + }); + + // Push modified block: load the block local via EP. + let modified_val = fun.get_local_from_ep(modified_block, ep, ep_offset, level, types::BasicObject); + let mut modified_args = vec![modified_val]; + if level == 0 { modified_args.push(modified_val); } + fun.push_insn(modified_block, Insn::Jump(BranchEdge { target: join_block, args: modified_args })); + + // Push unmodified block: inspect the current block handler to + // decide whether this path returns `nil` or `BlockParamProxy`. + let block_handler = fun.push_insn(unmodified_block, Insn::LoadField { recv: ep, id: FieldName::VM_ENV_DATA_INDEX_SPECVAL, offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, return_type: types::CInt64 }); + let original_local = if level == 0 { Some(state.getlocal(ep_offset)) } else { None }; + // `block_handler & 1 == 1` accepts both ISEQ (0b01) and ifunc + // (0b11) handlers. Keep a compile-time check that this shortcut + // does not accidentally accept symbol block handlers. + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + + + let profiled_block_summary = profiles.payload.profile.get_operand_types(exit_state.insn_idx) + .and_then(|types| types.first()) + .map(TypeDistributionSummary::new); + + let mut profiled_handlers = Vec::new(); + if let Some(summary) = profiled_block_summary.as_ref() { + if summary.is_monomorphic() || summary.is_polymorphic() || summary.is_skewed_polymorphic() { + for &profiled_type in summary.buckets() { + if profiled_type.is_empty() { + break; + } + if let Some(profiled_handler) = ProfiledBlockHandlerFamily::from_profiled_type(profiled_type) { + if !profiled_handlers.contains(&profiled_handler) { + profiled_handlers.push(profiled_handler); + } + } + } + } + } + + match profiled_handlers.as_slice() { + // No supported profiled families. Keep the generic fallback iseq/ifunc fallback + // for sites we do not specialize, such as no-profile and megamorphic sites. + [] => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + + // Bail out if the block handler is neither ISEQ nor ifunc + fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: SideExitReason::BlockParamProxyFallbackMiss, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![proxy_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + // A single supported profiled family. Emit a monomorphic fast path + [profiled_handler] => match profiled_handler { + ProfiledBlockHandlerFamily::Nil => { + fun.push_insn(unmodified_block, Insn::GuardBitEquals { val: block_handler, expected: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), reason: SideExitReason::BlockParamProxyNotNil, state: exit_id, recompile: None }); + let nil_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(Qnil) }); + let mut args = vec![nil_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + ProfiledBlockHandlerFamily::IseqOrIfunc => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + + // Bail out if the block handler is neither ISEQ nor ifunc + fun.push_insn(unmodified_block, Insn::GuardAnyBitSet { val: block_handler, mask: Const::CUInt64(0x1), mask_name: None, reason: SideExitReason::BlockParamProxyNotIseqOrIfunc, state: exit_id }); + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let proxy_val = fun.push_insn(unmodified_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![proxy_val]; + if let Some(local) = original_local { + args.push(local); + } + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args })); + } + }, + // Multiple supported profiled families. Emit a polymorphic dispatch + _ => { + let profiled_blocks = profiled_handlers.iter() + .map(|&kind| (kind, fun.new_block(branch_insn_idx))) + .collect::<Vec<_>>(); + + let mut current_block = unmodified_block; + + for &(kind, profiled_block) in &profiled_blocks { + match kind { + ProfiledBlockHandlerFamily::Nil => { + let none_handler = fun.push_insn(current_block, Insn::Const { + val: Const::CInt64(VM_BLOCK_HANDLER_NONE.into()), + }); + let is_none = fun.push_insn(current_block, Insn::IsBitEqual { + left: block_handler, + right: none_handler, + }); + + let next_block = fun.new_block(branch_insn_idx); + + fun.push_insn(current_block, Insn::CondBranch { + val: is_none, + if_true: BranchEdge { target: profiled_block, args: vec![] }, + if_false: BranchEdge { target: next_block, args: vec![] }, + }); + + current_block = next_block; + + let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(Qnil) }); + let mut args = vec![val]; + if let Some(local) = original_local { args.push(local); } + fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); + + } + ProfiledBlockHandlerFamily::IseqOrIfunc => { + // This handles two cases which are nearly identical. + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + let tag_mask = fun.push_insn(current_block, Insn::Const { val: Const::CInt64(0x1) }); + let tag_bits = fun.push_insn(current_block, Insn::IntAnd { + left: block_handler, + right: tag_mask, + }); + let is_iseq_or_ifunc = fun.push_insn(current_block, Insn::IsBitEqual { + left: tag_bits, + right: tag_mask, + }); + let next_block = fun.new_block(branch_insn_idx); + fun.push_insn(current_block, Insn::CondBranch { + val: is_iseq_or_ifunc, + if_true: BranchEdge { target: profiled_block, args: vec![] }, + if_false: BranchEdge { target: next_block, args: vec![] }, + }); + current_block = next_block; + + // TODO(Shopify/ruby#753): GC root, so we should be able to avoid unnecessary GC tracing + let val = fun.push_insn(profiled_block, Insn::Const { val: Const::Value(unsafe { rb_block_param_proxy }) }); + let mut args = vec![val]; + if let Some(local) = original_local { args.push(local); } + fun.push_insn(profiled_block, Insn::Jump(BranchEdge { target: join_block, args })); + }, + } + } + + fun.push_insn(current_block, Insn::SideExit { state: exit_id, reason: SideExitReason::BlockParamProxyProfileNotCovered, recompile: None }); + } + } + + // Continue compilation from the merged continuation block at the next + // instruction. + if let Some(local_param) = join_local { + state.setlocal(ep_offset, local_param); + } + state.stack_push(join_result); + block = join_block; + } + YARVINSN_getblockparam => { + let ep_offset = get_arg(pc, 0).as_u32(); + let level = get_arg(pc, 1).as_u32(); + let branch_insn_idx = exit_state.insn_idx as u32; + + // If the block param is already a Proc (modified), read it from EP. + // Otherwise, convert it to a Proc and store it to EP. + let modified_block = fun.new_block(branch_insn_idx); + let unmodified_block = fun.new_block(branch_insn_idx); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + let join_param = fun.push_insn(join_block, Insn::Param); + + let ep = fun.push_insn(block, Insn::GetEP { level }); + let flags = fun.load_ep_flags(block, ep); + let is_modified = fun.push_insn(block, Insn::IsBlockParamModified { flags }); + + fun.push_insn(block, Insn::CondBranch { + val: is_modified, + if_true: BranchEdge { target: modified_block, args: vec![] }, + if_false: BranchEdge { target: unmodified_block, args: vec![] } + }); + + // Push modified block: read Proc from EP. + let modified_val = fun.get_local_from_ep(modified_block, ep, ep_offset, level, types::BasicObject); + fun.push_insn(modified_block, Insn::Jump(BranchEdge { target: join_block, args: vec![modified_val] })); + + // Push unmodified block: convert block handler to Proc. + let unmodified_val = fun.push_insn(unmodified_block, Insn::GetBlockParam { + ep_offset, + level, + state: exit_id, + }); + fun.push_insn(unmodified_block, Insn::Jump(BranchEdge { target: join_block, args: vec![unmodified_val] })); + + // Continue compilation from the join block at the next instruction. + if level == 0 { + state.setlocal(ep_offset, join_param); + } + state.stack_push(join_param); + block = join_block; + } YARVINSN_pop => { state.stack_pop()?; } YARVINSN_dup => { state.stack_push(state.stack_top()?); } YARVINSN_dupn => { @@ -3188,79 +7820,84 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { n -= 1; } } - YARVINSN_opt_aref_with => { - // NB: opt_aref_with has an instruction argument for the call at get_arg(0) - let cd: *const rb_call_data = get_arg(pc, 1).as_ptr(); - let call_info = unsafe { rb_get_call_data_ci(cd) }; - if unknown_call_type(unsafe { rb_vm_ci_flag(call_info) }) { - // Unknown call type; side-exit into the interpreter - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownCallType }); - break; // End the block - } - let argc = unsafe { vm_ci_argc((*cd).ci) }; - - assert_eq!(1, argc, "opt_aref_with should only be emitted for argc=1"); - let aref_arg = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let args = vec![aref_arg]; - - let mut send_state = state.clone(); - send_state.stack_push(aref_arg); - let send_state = fun.push_insn(block, Insn::Snapshot { state: send_state }); - let recv = state.stack_pop()?; - let send = fun.push_insn(block, Insn::SendWithoutBlock { self_val: recv, cd, args, state: send_state }); - state.stack_push(send); - } YARVINSN_opt_neq => { // NB: opt_neq has two cd; get_arg(0) is for eq and get_arg(1) is for neq let cd: *const rb_call_data = get_arg(pc, 1).as_ptr(); let call_info = unsafe { rb_get_call_data_ci(cd) }; - if unknown_call_type(unsafe { rb_vm_ci_flag(call_info) }) { - // Unknown call type; side-exit into the interpreter - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownCallType }); + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); break; // End the block } - let argc = unsafe { vm_ci_argc((*cd).ci) }; + let argc = crate::profile::num_arguments_on_stack(cd); + assert_eq!(flags & VM_CALL_ARGS_BLOCKARG, 0); + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } let args = state.stack_pop_n(argc as usize)?; let recv = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let send = fun.push_insn(block, Insn::SendWithoutBlock { self_val: recv, cd, args, state: exit_id }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode) }); state.stack_push(send); } - YARVINSN_opt_hash_freeze | - YARVINSN_opt_ary_freeze | - YARVINSN_opt_str_freeze | + YARVINSN_opt_hash_freeze => { + let klass = HASH_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); + break; // End the block + } + } + YARVINSN_opt_ary_freeze => { + let klass = ARRAY_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); + break; // End the block + } + } + YARVINSN_opt_str_freeze => { + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_FREEZE; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); + break; // End the block + } + } YARVINSN_opt_str_uminus => { - // NB: these instructions have the recv for the call at get_arg(0) - let cd: *const rb_call_data = get_arg(pc, 1).as_ptr(); - let call_info = unsafe { rb_get_call_data_ci(cd) }; - if unknown_call_type(unsafe { rb_vm_ci_flag(call_info) }) { - // Unknown call type; side-exit into the interpreter - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownCallType }); + let klass = STRING_REDEFINED_OP_FLAG; + let bop = BOP_UMINUS; + if unsafe { rb_BASIC_OP_UNREDEFINED_P(bop, klass) } { + fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::BOPRedefined { klass, bop }, state: exit_id }); + let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); + state.stack_push(recv); + } else { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::PatchPoint(Invariant::BOPRedefined { klass, bop }), recompile: None }); break; // End the block } - let argc = unsafe { vm_ci_argc((*cd).ci) }; - let name = insn_name(opcode as usize); - assert_eq!(0, argc, "{name} should not have args"); - let args = vec![]; - - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let recv = fun.push_insn(block, Insn::Const { val: Const::Value(get_arg(pc, 0)) }); - let send = fun.push_insn(block, Insn::SendWithoutBlock { self_val: recv, cd, args, state: exit_id }); - state.stack_push(send); } - YARVINSN_leave => { - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); fun.push_insn(block, Insn::CheckInterrupts { state: exit_id }); fun.push_insn(block, Insn::Return { val: state.stack_pop()? }); break; // Don't enqueue the next block as a successor } YARVINSN_throw => { - fun.push_insn(block, Insn::Throw { throw_state: get_arg(pc, 0).as_u32(), val: state.stack_pop()? }); + fun.push_insn(block, Insn::Throw { throw_state: get_arg(pc, 0).as_u32(), val: state.stack_pop()?, state: exit_id }); break; // Don't enqueue the next block as a successor } @@ -3293,63 +7930,506 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { YARVINSN_opt_send_without_block => { let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); let call_info = unsafe { rb_get_call_data_ci(cd) }; - if unknown_call_type(unsafe { rb_vm_ci_flag(call_info) }) { - // Unknown call type; side-exit into the interpreter - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownCallType }); + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); break; // End the block } - let argc = unsafe { vm_ci_argc((*cd).ci) }; + let argc = crate::profile::num_arguments_on_stack(cd); + let mid = unsafe { rb_vm_ci_mid(call_info) }; + + // Check for calls to directives + if argc == 0 + && (mid == ID!(induce_side_exit_bang) || mid == ID!(induce_compile_failure_bang) || mid == ID!(induce_breakpoint_bang)) + && fun.type_of(state.stack_top()?) + .ruby_object() + .is_some_and(|obj| obj == VALUE(state::ZJIT_MODULE.load(Ordering::Relaxed))) + { + + if mid == ID!(induce_side_exit_bang) + && state::zjit_module_method_match_serial(ID!(induce_side_exit_bang), &state::INDUCE_SIDE_EXIT_SERIAL) + { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::DirectiveInduced, recompile: None }); + break; // End the block + } + if mid == ID!(induce_compile_failure_bang) + && state::zjit_module_method_match_serial(ID!(induce_compile_failure_bang), &state::INDUCE_COMPILE_FAILURE_SERIAL) + { + return Err(ParseError::DirectiveInduced); + } + if mid == ID!(induce_breakpoint_bang) + && state::zjit_module_method_match_serial(ID!(induce_breakpoint_bang), &state::INDUCE_BREAKPOINT_SERIAL) + { + fun.push_insn(block, Insn::BreakPoint); + state.stack_pop()?; // pop the receiver (::RubyVM::ZJIT) + state.stack_push(fun.push_insn(block, Insn::Const { val: Const::Value(Qnil) })); + } + } + + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + + { + fn new_branch_block( + fun: &mut Function, + cd: *const rb_call_data, + argc: usize, + opcode: u32, + new_type: Type, + insn_idx: u32, + exit_state: &FrameState, + locals_count: usize, + stack_count: usize, + join_block: BlockId, + ) -> BlockId { + let block = fun.new_block(insn_idx); + let self_param = fun.push_insn(block, Insn::Param); + let mut state = exit_state.clone(); + state.locals.clear(); + state.stack.clear(); + state.locals.extend((0..locals_count).map(|_| fun.push_insn(block, Insn::Param))); + state.stack.extend((0..stack_count).map(|_| fun.push_insn(block, Insn::Param))); + let snapshot = fun.push_insn(block, Insn::Snapshot { state: state.clone() }); + let args = state.stack_pop_n(argc).unwrap(); + let recv = state.stack_pop().unwrap(); + let refined_recv = fun.push_insn(block, Insn::RefineType { val: recv, new_type }); + state.replace(recv, refined_recv); + let send = fun.push_insn(block, Insn::Send { recv: refined_recv, cd, block: None, args, state: snapshot, reason: Uncategorized(opcode) }); + state.stack_push(send); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: state.as_args(self_param) })); + block + } + let branch_insn_idx = exit_state.insn_idx as u32; + let locals_count = state.locals.len(); + let stack_count = state.stack.len(); + let recv = state.stack_topn(argc as usize)?; // args are on top + let entry_args = state.as_args(self_param); + if let Some(summary) = fun.polymorphic_summary(&profiles, recv, exit_state.insn_idx) { + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + // Dedup by expected type so immediate/heap variants + // under the same Ruby class can still get separate branches. + let mut seen_types = Vec::with_capacity(summary.buckets().len()); + for &profiled_type in summary.buckets() { + if profiled_type.is_empty() { break; } + let expected = Type::from_profiled_type(profiled_type); + if seen_types.iter().any(|ty: &Type| ty.bit_equal(expected)) { + continue; + } + seen_types.push(expected); + let has_type = fun.push_insn(block, Insn::HasType { val: recv, expected }); + let iftrue_block = + new_branch_block(&mut fun, cd, argc as usize, opcode, expected, branch_insn_idx, &exit_state, locals_count, stack_count, join_block); + let target = BranchEdge { target: iftrue_block, args: entry_args.clone() }; + let fall_through = fun.new_block(insn_idx); + fun.push_insn(block, Insn::CondBranch { + val: has_type, + if_true: target, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + block = fall_through; + } + // Continue compilation from the join block at the next instruction. + // Make a copy of the current state without the args (pop the receiver + // and push the result) because we just use the locals/stack sizes to + // make the right number of Params + let mut join_state = state.clone(); + join_state.stack_pop_n(argc as usize)?; + queue.push_back((join_state, join_block, insn_idx, local_inval)); + // In the fallthrough case, do a generic interpreter send and then join. + let args = state.stack_pop_n(argc as usize)?; + let recv = state.stack_pop()?; + let reason = SendWithoutBlockPolymorphicFallback; + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason }); + state.stack_push(send); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: state.as_args(self_param) })); + break; // End the block + } + } let args = state.stack_pop_n(argc as usize)?; let recv = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let send = fun.push_insn(block, Insn::SendWithoutBlock { self_val: recv, cd, args, state: exit_id }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode) }); state.stack_push(send); } YARVINSN_send => { let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); let call_info = unsafe { rb_get_call_data_ci(cd) }; - if unknown_call_type(unsafe { rb_vm_ci_flag(call_info) }) { - // Unknown call type; side-exit into the interpreter - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownCallType }); + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); break; // End the block } - let argc = unsafe { vm_ci_argc((*cd).ci) }; + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; - let args = state.stack_pop_n(argc as usize)?; + let args = state.stack_pop_n(crate::profile::num_arguments_on_stack(cd))?; let recv = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let send = fun.push_insn(block, Insn::Send { self_val: recv, cd, blockiseq, args, state: exit_id }); + let block_handler = if !blockiseq.is_null() { + Some(BlockHandler::BlockIseq(blockiseq)) + } else if block_arg { + Some(BlockHandler::BlockArg) + } else { + None + }; + let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, state: exit_id, reason: Uncategorized(opcode) }); state.stack_push(send); + + if let Some(BlockHandler::BlockIseq(_)) = block_handler { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_sendforward => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + let forwarding = (flags & VM_CALL_FORWARDING) != 0; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle the call type; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + + let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; + let recv = state.stack_pop()?; + let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: SendForwardNotSpecialized }); + state.stack_push(send_forward); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_invokesuper => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let args = state.stack_pop_n(crate::profile::num_arguments_on_stack(cd))?; + let recv = state.stack_pop()?; + let blockiseq: IseqPtr = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::InvokeSuper { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + state.stack_push(result); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_invokesuperforward => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let blockiseq: IseqPtr = get_arg(pc, 1).as_iseq(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + let forwarding = (flags & VM_CALL_FORWARDING) != 0; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let argc = unsafe { vm_ci_argc((*cd).ci) }; + let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; + let recv = state.stack_pop()?; + let result = fun.push_insn(block, Insn::InvokeSuperForward { recv, cd, blockiseq, args, state: exit_id, reason: InvokeSuperForwardNotSpecialized }); + state.stack_push(result); + + if !blockiseq.is_null() { + // Reload locals that may have been modified by the blockiseq. + // TODO: Avoid reloading locals that are not referenced by the blockiseq + // or not used after this. Max thinks we could eventually DCE them. + if !ep_escaped && !state.locals.is_empty() { + fun.gen_post_send_no_ep_escape_patch_point(block, &state, insn_idx); + } + let mut base: Option<InsnId> = None; + for local_idx in 0..state.locals.len() { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let recv = *base.get_or_insert_with(|| { + let base_insn = if !ep_escaped { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(block, base_insn) + }); + let val = if !ep_escaped { + fun.get_local_from_sp(block, recv, ep_offset_u32, types::BasicObject) + } else { + fun.get_local_from_ep(block, recv, ep_offset_u32, 0, types::BasicObject) + }; + state.setlocal(ep_offset_u32, val); + } + } + } + YARVINSN_invokeblock => { + let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); + let call_info = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(call_info) }; + if let Err(call_type) = unhandled_call_type(flags) { + // Can't handle tailcall; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledCallType(call_type), recompile: None }); + break; // End the block + } + // Side-exit send fallbacks while tracing to avoid FLAG_FINISH breaking throw TAG_RETURN semantics + if unsafe { rb_zjit_iseq_tracing_currently_enabled() } { + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::SendWhileTracing, recompile: None }); + break; + } + let args = state.stack_pop_n(crate::profile::num_arguments_on_stack(cd))?; + + // Check if this is a monomorphic IFUNC block handler we can specialize + let block_handler_types = profiles.payload.profile.get_operand_types(exit_state.insn_idx); + let is_ifunc = (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT)) == 0 + && block_handler_types.is_some_and(|types| types.len() == 1 && { + let summary = TypeDistributionSummary::new(&types[0]); + summary.is_monomorphic() && unsafe { rb_IMEMO_TYPE_P(summary.bucket(0).class(), imemo_ifunc) == 1 } + }); + + let result = if is_ifunc { + // Load the block handler from LEP + let level = get_lvar_level(fun.iseq); + let lep = fun.push_insn(block, Insn::GetEP { level }); + let block_handler = fun.push_insn(block, Insn::LoadField { + recv: lep, + id: FieldName::VM_ENV_DATA_INDEX_SPECVAL, + offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, + return_type: types::CInt64, + }); + + // Check IFUNC tag: (block_handler & 0x3) == 0x3 + let tag_mask = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); + let tag_bits = fun.push_insn(block, Insn::IntAnd { left: block_handler, right: tag_mask }); + let ifunc_tag = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); + let is_ifunc_match = fun.push_insn(block, Insn::IsBitEqual { left: tag_bits, right: ifunc_tag }); + + // Branch: on match, call InvokeBlockIfunc directly + let join_block = fun.new_block(insn_idx); + let join_param = fun.push_insn(join_block, Insn::Param); + let ifunc_block = fun.new_block(insn_idx); + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { + val: is_ifunc_match, + if_true: BranchEdge { target: ifunc_block, args: vec![] }, + if_false: BranchEdge { target: fall_through, args: vec![] }, + }); + + block = fall_through; + + let ifunc_result = fun.push_insn(ifunc_block, Insn::InvokeBlockIfunc { + cd, + block_handler, + args: args.clone(), + state: exit_id, + }); + fun.push_insn(ifunc_block, Insn::Jump(BranchEdge { target: join_block, args: vec![ifunc_result] })); + + // In the fallthrough case, use generic rb_vm_invokeblock and join + let fallback_result = fun.push_insn(block, Insn::InvokeBlock { + cd, args, state: exit_id, reason: InvokeBlockNotSpecialized, + }); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![fallback_result] })); + + // Continue compilation from the join block + block = join_block; + join_param + } else { + fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: InvokeBlockNotSpecialized }) + }; + state.stack_push(result); } YARVINSN_getglobal => { let id = ID(get_arg(pc, 0).as_u64()); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let result = fun.push_insn(block, Insn::GetGlobal { id, state: exit_id }); state.stack_push(result); } YARVINSN_setglobal => { let id = ID(get_arg(pc, 0).as_u64()); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let val = state.stack_pop()?; fun.push_insn(block, Insn::SetGlobal { id, val, state: exit_id }); } YARVINSN_getinstancevariable => { let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); // ic is in arg 1 - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - let result = fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, state: exit_id }); - state.stack_push(result); + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_getivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_getivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); + break; // End the block + } + if let Some(summary) = fun.polymorphic_summary(&profiles, self_param, exit_state.insn_idx) { + self_param = fun.push_insn(block, Insn::GuardType { val: self_param, guard_type: types::HeapBasicObject, state: exit_id, recompile: None }); + let rbasic_flags = fun.load_rbasic_flags(block, self_param); + let join_block = insn_idx_to_block.get(&insn_idx).copied().unwrap_or_else(|| fun.new_block(insn_idx)); + let join_param = fun.push_insn(join_block, Insn::Param); + // Dedup by expected shape so objects with different classes but the same shape can share code + // TODO(max): De-duplicate further by checking ivar offsets to allow + // different shapes with the same ivar layout to share code + let mut seen_shape_and_flags = Vec::with_capacity(summary.buckets().len()); + for &profiled_type in summary.buckets() { + // End of the buckets + if profiled_type.is_empty() { break; } + // Instance variable lookups on immediate values are always nil; don't bother + if profiled_type.flags().is_immediate() { continue; } + let expected_shape = profiled_type.shape(); + let (expected_rbasic_flags, rbasic_flags_mask) = profiled_type.rbasic_flags_and_mask(); + assert!(expected_shape.is_valid()); + // Too-complex shapes use hash tables for ivars; + // rb_shape_get_iv_index doesn't work for them. + // Let the fallthrough GetIvar handle these. + if expected_shape.is_complex() { continue; } + if seen_shape_and_flags.contains(&expected_rbasic_flags) { continue; } + seen_shape_and_flags.push(expected_rbasic_flags); + let rbasic_flags_mask = fun.push_insn(block, Insn::Const { val: Const::CUInt64(rbasic_flags_mask) }); + // The expected shape can change over run, so we put it + // as a pointer to keep it stable in snapshot tests. + let expected_rbasic_flags = fun.push_insn(block, Insn::Const { val: Const::CPtr(ptr::without_provenance(expected_rbasic_flags.to_usize())) }); + let expected_rbasic_flags = fun.push_insn(block, Insn::RefineType { val: expected_rbasic_flags, new_type: types::CUInt64 }); + let masked = fun.push_insn(block, Insn::IntAnd { left: rbasic_flags, right: rbasic_flags_mask}); + let has_shape_and_type = fun.push_insn(block, Insn::IsBitEqual { left: masked, right: expected_rbasic_flags }); + let iftrue_block = fun.new_block(insn_idx); + let target = BranchEdge { target: iftrue_block, args: vec![] }; + let fall_through = fun.new_block(insn_idx); + + fun.push_insn(block, Insn::CondBranch { val: has_shape_and_type, + if_true: target, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + + block = fall_through; + let result = fun.load_ivar(iftrue_block, self_param, profiled_type, id, exit_id); + fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![result] })); + } + // In the fallthrough case, do a generic interpreter getivar and then join. + let result = fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic, state: exit_id }); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![result] })); + state.stack_push(join_param); + // Continue compilation from the join block at the next instruction. + // Make a copy of the current state without the args (pop the receiver + // and push the result) because we just use the locals/stack sizes to + // make the right number of Params + block = join_block; + } else { + // Possibly monomorphic case; handled in optimize_getivar + let result = fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic, state: exit_id }); + state.stack_push(result); + } } YARVINSN_setinstancevariable => { let id = ID(get_arg(pc, 0).as_u64()); - // ic is in arg 1 - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); + let ic = get_arg(pc, 1).as_ptr(); + // Assume single-Ractor mode to omit gen_prepare_non_leaf_call on gen_setivar + // TODO: We only really need this if self_val is a class/module + if !fun.assume_single_ractor_mode(block, exit_id) { + // gen_setivar assumes single Ractor; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); + break; // End the block + } let val = state.stack_pop()?; - fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, val, state: exit_id }); + fun.push_insn(block, Insn::SetIvar { self_val: self_param, id, ic, val, state: exit_id }); + // SetIvar will raise if self is an immediate. If it raises, we will have + // exited JIT code. So upgrade the type within JIT code to a heap object. + self_param = fun.push_insn(block, Insn::RefineType { val: self_param, new_type: types::HeapBasicObject }); + } + YARVINSN_getclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let result = fun.push_insn(block, Insn::GetClassVar { id, ic, state: exit_id }); + state.stack_push(result); + } + YARVINSN_setclassvariable => { + let id = ID(get_arg(pc, 0).as_u64()); + let ic = get_arg(pc, 1).as_ptr(); + let val = state.stack_pop()?; + fun.push_insn(block, Insn::SetClassVar { id, val, ic, state: exit_id }); } YARVINSN_opt_reverse => { // Reverse the order of the top N stack items. @@ -3365,7 +8445,6 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let flag = RangeType::from(get_arg(pc, 0).as_u32()); let high = state.stack_pop()?; let low = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let insn_id = fun.push_insn(block, Insn::NewRange { low, high, flag, state: exit_id }); state.stack_push(insn_id); } @@ -3379,17 +8458,20 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { args.push(self_param); args.reverse(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - // Check if this builtin is annotated let return_type = ZJITState::get_method_annotations() .get_builtin_properties(&bf) .map(|props| props.return_type); + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { bf, + recv: self_param, args, state: exit_id, + leaf, return_type, }); state.stack_push(insn_id); @@ -3405,33 +8487,30 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { args.push(local); } - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - // Check if this builtin is annotated let return_type = ZJITState::get_method_annotations() .get_builtin_properties(&bf) .map(|props| props.return_type); + let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; + let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; + let insn_id = fun.push_insn(block, Insn::InvokeBuiltin { bf, + recv: self_param, args, state: exit_id, + leaf, return_type, }); state.stack_push(insn_id); } YARVINSN_objtostring => { let cd: *const rb_call_data = get_arg(pc, 0).as_ptr(); - let call_info = unsafe { rb_get_call_data_ci(cd) }; - - if unknown_call_type(unsafe { rb_vm_ci_flag(call_info) }) { - assert!(false, "objtostring should not have unknown call type"); - } - let argc = unsafe { vm_ci_argc((*cd).ci) }; + let argc = crate::profile::num_arguments_on_stack(cd); assert_eq!(0, argc, "objtostring should not have args"); let recv = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let objtostring = fun.push_insn(block, Insn::ObjToString { val: recv, cd, state: exit_id }); state.stack_push(objtostring) } @@ -3439,7 +8518,6 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let str = state.stack_pop()?; let val = state.stack_pop()?; - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); let anytostring = fun.push_insn(block, Insn::AnyToString { val, str, state: exit_id }); state.stack_push(anytostring); } @@ -3447,11 +8525,9 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { let key = get_arg(pc, 0).as_u64(); let svar = get_arg(pc, 1).as_u64(); - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - if svar == 0 { // TODO: Handle non-backref - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownSpecialVariable(key) }); + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownSpecialVariable(key), recompile: None }); // End the block break; } else if svar & 0x01 != 0 { @@ -3466,10 +8542,33 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { state.stack_push(result); } } + YARVINSN_expandarray => { + let num = get_arg(pc, 0).as_u64(); + let flag = get_arg(pc, 1).as_u64(); + if flag != 0 { + // We don't (yet) handle 0x01 (rest args), 0x02 (post args), or 0x04 + // (reverse?) + // + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); + break; // End the block + } + let val = state.stack_pop()?; + let array = fun.push_insn(block, Insn::GuardType { val, guard_type: types::ArrayExact, state: exit_id, recompile: None }); + let length = fun.push_insn(block, Insn::ArrayLength { array }); + let expected = fun.push_insn(block, Insn::Const { val: Const::CInt64(num as i64) }); + fun.push_insn(block, Insn::GuardGreaterEq { left: length, right: expected, reason: SideExitReason::ExpandArray, state: exit_id }); + for i in (0..num).rev() { + // We do not emit a length guard here because in-bounds is already + // ensured by the expandarray length check above. + let index = fun.push_insn(block, Insn::Const { val: Const::CInt64(i.try_into().unwrap()) }); + let element = fun.push_insn(block, Insn::ArrayAref { array, index }); + state.stack_push(element); + } + } _ => { - // Unknown opcode; side-exit into the interpreter - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state }); - fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnknownOpcode(opcode) }); + // Unhandled opcode; side-exit into the interpreter + fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::UnhandledYARVInsn(opcode), recompile: None }); break; // End the block } } @@ -3477,12 +8576,16 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { if insn_idx_to_block.contains_key(&insn_idx) { let target = insn_idx_to_block[&insn_idx]; fun.push_insn(block, Insn::Jump(BranchEdge { target, args: state.as_args(self_param) })); - queue.push_back((state, target, insn_idx)); + queue.push_back((state, target, insn_idx, local_inval)); break; // End the block } } } + // Populate the entries superblock with an Entries instruction targeting all entry blocks + fun.seal_entries(); + + fun.set_param_types(); fun.infer_types(); match get_option!(dump_hir_init) { @@ -3493,12 +8596,424 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result<Function, ParseError> { } fun.profiles = Some(profiles); - if let Err(e) = fun.validate() { - return Err(ParseError::Validation(e)); + if let Err(err) = crate::stats::trace_compile_phase("validate", || fun.validate()) { + debug!("ZJIT: {err:?}: Initial HIR:\n{}", FunctionPrinter::without_snapshot(&fun)); + return Err(ParseError::Validation(err)); } Ok(fun) } +/// Compile an entry_block for the interpreter +fn compile_entry_block(fun: &mut Function, jit_entry_insns: &[u32], insn_idx_to_block: &HashMap<u32, BlockId>) { + let mut entry_block = fun.entry_block; + let (self_param, entry_state) = compile_entry_state(fun); + let mut pc: Option<InsnId> = None; + let &all_opts_passed_insn_idx = jit_entry_insns.last().unwrap(); + + // Check-and-jump for each missing optional PC + let mut iter = jit_entry_insns.iter().peekable(); + while let Some(&jit_entry_insn) = iter.next() { + if jit_entry_insn == all_opts_passed_insn_idx { + continue; + } + let target_block = insn_idx_to_block.get(&jit_entry_insn) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + // Load PC once at the start of the block, shared among all cases + let pc = *pc.get_or_insert_with(|| fun.push_insn(entry_block, Insn::LoadPC)); + let expected_pc = fun.push_insn(entry_block, Insn::Const { + val: Const::CPtr(unsafe { rb_iseq_pc_at_idx(fun.iseq, jit_entry_insn) } as *const u8), + }); + let test_id = fun.push_insn(entry_block, Insn::IsBitEqual { left: pc, right: expected_pc }); + + let next_insn_idx = **iter.peek().expect("last entry is skipped so there is always a next"); + let fall_through = fun.new_block(next_insn_idx); + + fun.push_insn(entry_block, Insn::CondBranch { + val: test_id, + if_true: BranchEdge { target: target_block, args: entry_state.as_args(self_param) }, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + entry_block = fall_through; + } + + // Terminate the block with a jump to the block with all optionals passed + let target_block = insn_idx_to_block.get(&all_opts_passed_insn_idx) + .copied() + .expect("we make a block for each jump target and \ + each entry in the ISEQ opt_table is a jump target"); + fun.push_insn(entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile initial locals for an entry_block for the interpreter +fn compile_entry_state(fun: &mut Function) -> (InsnId, FrameState) { + let entry_block = fun.entry_block; + fun.push_insn(entry_block, Insn::EntryPoint { jit_entry_idx: None }); + + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let rest_param_idx = iseq_rest_param_idx(params); + + let self_param = fun.push_insn(entry_block, Insn::LoadSelf); + let mut entry_state = FrameState::new(iseq); + // If the ISEQ does not escape EP, we can assume EP + 1 == SP + // TODO: This should maybe also consider if the EP has historically been escaped in this iseq. + // (see: https://github.com/Shopify/ruby/issues/774) + let use_sp = !iseq_escapes_ep(iseq); + let mut base: Option<InsnId> = None; + for local_idx in 0..num_locals(iseq) { + if local_idx < param_size { + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let return_type = if Some(local_idx as i32) == rest_param_idx { + types::ArrayExact + } else { + types::BasicObject + }; + let recv = *base.get_or_insert_with(|| { + let base_insn = if use_sp { Insn::LoadSP } else { Insn::GetEP { level: 0 } }; + fun.push_insn(entry_block, base_insn) + }); + let val = if use_sp { + fun.get_local_from_sp(entry_block, recv, ep_offset_u32, return_type) + } else { + fun.get_local_from_ep(entry_block, recv, ep_offset_u32, 0, return_type) + }; + entry_state.locals.push(val); + } else { + entry_state.locals.push(fun.push_insn(entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +/// Compile a jit_entry_block +fn compile_jit_entry_block(fun: &mut Function, jit_entry_idx: usize, target_block: BlockId) { + let jit_entry_block = fun.jit_entry_blocks[jit_entry_idx]; + fun.push_insn(jit_entry_block, Insn::EntryPoint { jit_entry_idx: Some(jit_entry_idx) }); + + // Prepare entry_state with basic block params + let (self_param, entry_state) = compile_jit_entry_state(fun, jit_entry_block, jit_entry_idx); + + if get_option!(stats) { + fun.count_iseq_calls(jit_entry_block); + } + // Jump to target_block + fun.push_insn(jit_entry_block, Insn::Jump(BranchEdge { target: target_block, args: entry_state.as_args(self_param) })); +} + +/// Compile params and initial locals for a jit_entry_block +fn compile_jit_entry_state(fun: &mut Function, jit_entry_block: BlockId, jit_entry_idx: usize) -> (InsnId, FrameState) { + let iseq = fun.iseq; + let params = unsafe { iseq.params() }; + let param_size = params.size.to_usize(); + let opt_num: usize = params.opt_num.try_into().expect("iseq param opt_num >= 0"); + let lead_num: usize = params.lead_num.try_into().expect("iseq param lead_num >= 0"); + let passed_opt_num = jit_entry_idx; + + // If the iseq has keyword parameters, the keyword bits local will be appended to the local table. + let kw_bits_idx: Option<usize> = if unsafe { rb_get_iseq_flags_has_kw(iseq) } { + let keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; + if !keyword.is_null() { + Some(unsafe { (*keyword).bits_start } as usize) + } else { + None + } + } else { + None + }; + + let mut arg_idx: u32 = 0; + // For `def` methods on classes that can only produce heap (non-immediate) + // instances, `self` is a HeapBasicObject. See `iseq_self_is_heap_object`. + let self_type = if fun.self_is_heap_object { types::HeapBasicObject } else { types::BasicObject }; + let self_param = fun.push_insn(jit_entry_block, Insn::LoadArg { idx: arg_idx, id: FieldName::SelfParam, val_type: self_type }); + arg_idx += 1; + let mut entry_state = FrameState::new(iseq); + let mut ep: Option<InsnId> = None; + for local_idx in 0..num_locals(iseq) { + if (lead_num + passed_opt_num..lead_num + opt_num).contains(&local_idx) { + // Omitted optionals are locals, so they start as nils before their code run + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } else if Some(local_idx) == kw_bits_idx { + // Read the kw_bits value written by the caller to the callee frame. + // This tells us which optional keywords were NOT provided and need their defaults evaluated. + // Note: The caller writes kw_bits to memory via gen_send_iseq_direct but does NOT pass it + // as a C argument, so we must read it from EP memory rather than Param. + let ep_offset = local_idx_to_ep_offset(iseq, local_idx); + let ep_offset_u32 = u32::try_from(ep_offset) + .unwrap_or_else(|_| panic!("Could not convert ep_offset {ep_offset} to u32")); + let ep = *ep.get_or_insert_with(|| fun.push_insn(jit_entry_block, Insn::GetEP { level: 0 })); + entry_state.locals.push(fun.get_local_from_ep( + jit_entry_block, + ep, + ep_offset_u32, + 0, + types::BasicObject, + )); + } else if local_idx < param_size { + let id = unsafe { rb_zjit_local_id(iseq, local_idx.try_into().unwrap()) }; + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::LoadArg { idx: arg_idx, id: id.into(), val_type: types::BasicObject })); + arg_idx += 1; + } else { + entry_state.locals.push(fun.push_insn(jit_entry_block, Insn::Const { val: Const::Value(Qnil) })); + } + } + (self_param, entry_state) +} + +pub struct Dominators { + /// Immediate dominator for each block, indexed by BlockId. + /// idom(root) = root (self-loop is sentinel), idom[unreachable] == IDOM_NONE. + idoms: Vec<BlockId>, +} + +/// Sentinel value for "no idom computed yet". +const IDOM_NONE: BlockId = BlockId(usize::MAX); + +impl Dominators { + pub fn new(f: &Function) -> Self { + let mut cfi = ControlFlowInfo::new(f); + Self::with_cfi(f, &mut cfi) + } + + /// Compute immediate dominators using the "engineered algorithm" from + /// Cooper, Harvey & Kennedy, "A Simple, Fast Dominance Algorithm" (2001), + /// Figure 3: <https://www.cs.tufts.edu/~nr/cs257/archive/keith-cooper/dom14.pdf> + pub fn with_cfi(f: &Function, cfi: &mut ControlFlowInfo) -> Self { + let rpo = f.reverse_post_order(); + let num_blocks = f.blocks.len(); + + // Map BlockId -> RPO index for O(1) lookup in intersect. + let mut rpo_order = vec![usize::MAX; num_blocks]; + for (idx, &block) in rpo.iter().enumerate() { + rpo_order[block.0] = idx; + } + + // Initialize idom: root's idom is itself, everything else is undefined. + let mut idoms = vec![IDOM_NONE; num_blocks]; + let root = f.entries_block; + idoms[root.0] = root; + + let mut changed = true; + while changed { + changed = false; + for &block in &rpo { + if block == root { continue; } + + // Find the first predecessor that already has an idom computed. + let preds: Vec<BlockId> = cfi.predecessors(block).collect(); + let mut new_idom = IDOM_NONE; + for &p in &preds { + if idoms[p.0] != IDOM_NONE { + new_idom = p; + break; + } + } + if new_idom == IDOM_NONE { continue; } + + // Intersect with remaining processed predecessors. + for &p in &preds { + if p == new_idom { continue; } + if idoms[p.0] != IDOM_NONE { + new_idom = Self::intersect(&idoms, &rpo_order, p, new_idom); + } + } + + if idoms[block.0] != new_idom { + idoms[block.0] = new_idom; + changed = true; + } + } + } + + Self { idoms } + } + + /// Walk up the dominator tree from two fingers until they meet. + /// Uses RPO indices: a node with a *lower* RPO index is *higher* in the tree. + fn intersect(idoms: &[BlockId], rpo_order: &[usize], mut b1: BlockId, mut b2: BlockId) -> BlockId { + while b1 != b2 { + while rpo_order[b1.0] > rpo_order[b2.0] { + b1 = idoms[b1.0]; + } + while rpo_order[b2.0] > rpo_order[b1.0] { + b2 = idoms[b2.0]; + } + } + b1 + } + + /// Return the immediate dominator of `block`. + pub fn idom(&self, block: BlockId) -> BlockId { + self.idoms[block.0] + } + + /// Return true if `left` is dominated by `right`. + pub fn is_dominated_by(&self, left: BlockId, right: BlockId) -> bool { + if self.idom(left) == IDOM_NONE { return false; } + let mut block = left; + loop { + if block == right { return true; } + if self.idom(block) == block { return false; } + block = self.idom(block); + } + } + + /// Compute the full dominator set for `block` by walking the idom chain to the root. + /// Returns dominators sorted by BlockId (ascending). Only used in tests; + /// production code should use `idom()` or `is_dominated_by()` instead. + pub fn dominators(&self, block: BlockId) -> Vec<BlockId> { + let mut doms = Vec::new(); + if self.idom(block) != IDOM_NONE { + let mut b = block; + loop { + doms.push(b); + if self.idom(b) == b { break; } + b = self.idom(b); + } + } + doms.sort(); + doms + } +} + +pub struct ControlFlowInfo<'a> { + function: &'a Function, + successor_map: HashMap<BlockId, Vec<BlockId>>, + predecessor_map: HashMap<BlockId, Vec<BlockId>>, +} + +impl<'a> ControlFlowInfo<'a> { + pub fn new(function: &'a Function) -> Self { + let mut successor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + let mut predecessor_map: HashMap<BlockId, Vec<BlockId>> = HashMap::new(); + + for block_id in function.reverse_post_order() { + let mut successors = function.successors(block_id); + successors.dedup(); + + // Update predecessors for successor blocks. + for &succ_id in &successors { + predecessor_map + .entry(succ_id) + .or_default() + .push(block_id); + } + + // Store successors for this block. + successor_map.insert(block_id, successors); + } + + Self { + function, + successor_map, + predecessor_map, + } + } + + pub fn is_succeeded_by(&self, left: BlockId, right: BlockId) -> bool { + self.successor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn is_preceded_by(&self, left: BlockId, right: BlockId) -> bool { + self.predecessor_map.get(&right).is_some_and(|set| set.contains(&left)) + } + + pub fn predecessors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.predecessor_map.get(&block).into_iter().flatten().copied() + } + + pub fn successors(&self, block: BlockId) -> impl Iterator<Item = BlockId> { + self.successor_map.get(&block).into_iter().flatten().copied() + } +} + +pub struct LoopInfo<'a> { + cfi: &'a ControlFlowInfo<'a>, + dominators: &'a Dominators, + loop_depths: HashMap<BlockId, u32>, + loop_headers: BlockSet, + back_edge_sources: BlockSet, +} + +impl<'a> LoopInfo<'a> { + pub fn new(cfi: &'a ControlFlowInfo<'a>, dominators: &'a Dominators) -> Self { + let mut loop_headers: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let mut loop_depths: HashMap<BlockId, u32> = HashMap::new(); + let mut back_edge_sources: BlockSet = BlockSet::with_capacity(cfi.function.num_blocks()); + let rpo = cfi.function.reverse_post_order(); + + for &block in &rpo { + loop_depths.insert(block, 0); + } + + // Collect loop headers. + for &block in &rpo { + // Initialize the loop depths. + for predecessor in cfi.predecessors(block) { + if dominators.is_dominated_by(predecessor, block) { + // Found a loop header, so then identify the natural loop. + loop_headers.insert(block); + back_edge_sources.insert(predecessor); + let loop_blocks = Self::find_natural_loop(cfi, block, predecessor); + // Increment the loop depth. + for loop_block in &loop_blocks { + *loop_depths.get_mut(loop_block).expect("Loop block should be populated.") += 1; + } + } + } + } + + Self { + cfi, + dominators, + loop_depths, + loop_headers, + back_edge_sources, + } + } + + fn find_natural_loop( + cfi: &ControlFlowInfo, + header: BlockId, + back_edge_source: BlockId, + ) -> HashSet<BlockId> { + // todo(aidenfoxivey): Reimplement using BlockSet + let mut loop_blocks = HashSet::new(); + let mut stack = vec![back_edge_source]; + + loop_blocks.insert(header); + loop_blocks.insert(back_edge_source); + + while let Some(block) = stack.pop() { + for pred in cfi.predecessors(block) { + // Pushes to stack only if `pred` wasn't already in `loop_blocks`. + if loop_blocks.insert(pred) { + stack.push(pred) + } + } + } + + loop_blocks + } + + pub fn loop_depth(&self, block: BlockId) -> u32 { + self.loop_depths.get(&block).copied().unwrap_or(0) + } + + pub fn is_back_edge_source(&self, block: BlockId) -> bool { + self.back_edge_sources.get(block) + } + + pub fn is_loop_header(&self, block: BlockId) -> bool { + self.loop_headers.get(block) + } +} + #[cfg(test)] mod union_find_tests { use super::UnionFind; @@ -3517,6 +9032,13 @@ mod union_find_tests { } #[test] + fn test_find_halts_with_identity_make_equal_to() { + let mut uf = UnionFind::<usize>::new(); + uf.make_equal_to(0, 0); + assert_eq!(uf.find(0), 0); + } + + #[test] fn test_find_returns_transitive_target() { let mut uf = UnionFind::new(); uf.make_equal_to(3, 4); @@ -3543,59 +9065,75 @@ mod rpo_tests { #[test] fn one_block() { let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; let entry = function.entry_block; let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); function.push_insn(entry, Insn::Return { val }); - assert_eq!(function.rpo(), vec![entry]); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry]); } #[test] fn jump() { let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; let entry = function.entry_block; let exit = function.new_block(0); function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); - let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn(entry, Insn::Return { val }); - assert_eq!(function.rpo(), vec![entry, exit]); + let val = function.push_insn(exit, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry, exit]); } #[test] fn diamond_iftrue() { let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; let entry = function.entry_block; let side = function.new_block(0); let exit = function.new_block(0); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn(entry, Insn::IfTrue { val, target: BranchEdge { target: side, args: vec![] } }); - function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); - let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn(entry, Insn::Return { val }); - assert_eq!(function.rpo(), vec![entry, side, exit]); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: side, args: vec![] }, + if_false: BranchEdge { target: exit, args: vec![] } + }); + let val = function.push_insn(exit, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry, side, exit]); } #[test] fn diamond_iffalse() { let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; let entry = function.entry_block; let side = function.new_block(0); let exit = function.new_block(0); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); - function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); - let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn(entry, Insn::Return { val }); - assert_eq!(function.rpo(), vec![entry, side, exit]); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); + let val = function.push_insn(exit, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry, side, exit]); } #[test] fn a_loop() { let mut function = Function::new(std::ptr::null()); + let entries = function.entries_block; let entry = function.entry_block; function.push_insn(entry, Insn::Jump(BranchEdge { target: entry, args: vec![] })); - assert_eq!(function.rpo(), vec![entry]); + function.seal_entries(); + assert_eq!(function.reverse_post_order(), vec![entries, entry]); } } @@ -3609,7 +9147,7 @@ mod validation_tests { Err(validation_err) => { assert_eq!(validation_err, expected); } - Ok(_) => assert!(false, "Expected validation error"), + Ok(_) => panic!("Expected validation error"), } } @@ -3618,6 +9156,7 @@ mod validation_tests { let mut function = Function::new(std::ptr::null()); let entry = function.entry_block; function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.seal_entries(); assert_matches_err(function.validate(), ValidationError::BlockHasNoTerminator(entry)); } @@ -3628,6 +9167,8 @@ mod validation_tests { let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); let insn_id = function.push_insn(entry, Insn::Return { val }); function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::Unreachable); + function.seal_entries(); assert_matches_err(function.validate(), ValidationError::TerminatorNotAtEnd(entry, insn_id, 1)); } @@ -3637,7 +9178,15 @@ mod validation_tests { let entry = function.entry_block; let side = function.new_block(0); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn(entry, Insn::IfTrue { val, target: BranchEdge { target: side, args: vec![val, val, val] } }); + let fall_through = function.new_block(1); + function.push_insn(fall_through, Insn::Unreachable); + function.push_insn(side, Insn::Unreachable); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: side, args: vec![val, val, val] }, + if_false: BranchEdge { target: fall_through, args: vec![] } + }); + function.seal_entries(); assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); } @@ -3647,7 +9196,15 @@ mod validation_tests { let entry = function.entry_block; let side = function.new_block(0); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![val, val, val] } }); + let fall_through = function.new_block(1); + function.push_insn(fall_through, Insn::Unreachable); + function.push_insn(side, Insn::Unreachable); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: fall_through, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![val, val, val] }, + }); + function.seal_entries(); assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); } @@ -3658,6 +9215,8 @@ mod validation_tests { let side = function.new_block(0); let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); function.push_insn(entry, Insn::Jump ( BranchEdge { target: side, args: vec![val, val, val] } )); + function.push_insn(side, Insn::Unreachable); + function.seal_entries(); assert_matches_err(function.validate(), ValidationError::MismatchedBlockArity(entry, 0, 3)); } @@ -3668,6 +9227,8 @@ mod validation_tests { // Create an instruction without making it belong to anything. let dangling = function.new_insn(Insn::Const{val: Const::CBool(true)}); let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: dangling, state: InsnId(0usize) }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, dangling)); } @@ -3679,6 +9240,8 @@ mod validation_tests { // Ret is a non-output instruction. let ret = function.push_insn(function.entry_block, Insn::Return { val: const_ }); let val = function.push_insn(function.entry_block, Insn::ArrayDup { val: ret, state: InsnId(0usize) }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(entry, val, ret)); } @@ -3692,9 +9255,16 @@ mod validation_tests { let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); let val1 = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); - function.push_insn(entry, Insn::IfFalse { val: val1, target: BranchEdge { target: side, args: vec![] } }); - function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + function.push_insn(entry, Insn::CondBranch { + val: val1, + if_true: BranchEdge { target: exit, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); let val2 = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + let const_ = function.push_insn(exit, Insn::Const{val: Const::CBool(true)}); + function.push_insn(exit, Insn::Return { val: const_ }); + + function.seal_entries(); crate::cruby::with_rubyvm(|| { function.infer_types(); assert_matches_err(function.validate_definite_assignment(), ValidationError::OperandNotDefined(exit, val2, v0)); @@ -3711,9 +9281,15 @@ mod validation_tests { let v0 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![] })); let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); - function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); - function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![] }, + if_false: BranchEdge { target: side, args: vec![] } + }); let _val = function.push_insn(exit, Insn::ArrayDup { val: v0, state: v0 }); + let const_ = function.push_insn(exit, Insn::Const{val: Const::CBool(true)}); + function.push_insn(exit, Insn::Return { val: const_ }); + function.seal_entries(); crate::cruby::with_rubyvm(|| { function.infer_types(); // Just checking that we don't panic. @@ -3724,35 +9300,116 @@ mod validation_tests { #[test] fn instruction_appears_twice_in_same_block() { let mut function = Function::new(std::ptr::null()); - let entry = function.entry_block; - let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - function.push_insn_id(entry, val); - function.push_insn(entry, Insn::Return { val }); - assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(entry, val)); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn_id(block, val); + function.push_insn(block, Insn::Return { val }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val)); } #[test] fn instruction_appears_twice_with_different_ids() { let mut function = Function::new(std::ptr::null()); - let entry = function.entry_block; - let val0 = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); - let val1 = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val0 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); + let val1 = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); function.make_equal_to(val1, val0); - function.push_insn(entry, Insn::Return { val: val0 }); - assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(entry, val0)); + function.push_insn(block, Insn::Return { val: val0 }); + function.seal_entries(); + assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(block, val0)); } #[test] fn instruction_appears_twice_in_different_blocks() { let mut function = Function::new(std::ptr::null()); - let entry = function.entry_block; - let val = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let block = function.new_block(0); + function.push_insn(function.entry_block, Insn::Jump(BranchEdge { target: block, args: vec![] })); + let val = function.push_insn(block, Insn::Const { val: Const::Value(Qnil) }); let exit = function.new_block(0); - function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![] })); + function.push_insn(block, Insn::Jump(BranchEdge { target: exit, args: vec![] })); function.push_insn_id(exit, val); function.push_insn(exit, Insn::Return { val }); + function.seal_entries(); assert_matches_err(function.validate(), ValidationError::DuplicateInstruction(exit, val)); } + + // The heap-fields pointer (`as_heap`, a CPtr) and the first embedded + // instance variable both live at ROBJECT_OFFSET_AS_HEAP_FIELDS == + // ROBJECT_OFFSET_AS_ARY == 0x10 on a Ruby object. They are distinct fields + // with incompatible value types that happen to share a base and an offset. + // Since we could end up with two `LoadField` on different shape types + // (e.g., as the result of inlining), `optimize_load_store` must not satisfy + // one load from another cached load with a different return type. The fault + // surfaces here as the forwarded value flowing into a `Return` with the + // wrong type (`CPtr` rather than `BasicObject`). + #[test] + fn optimize_load_store_does_not_alias_loads_with_incompatible_return_types() { + assert_eq!(ROBJECT_OFFSET_AS_HEAP_FIELDS, ROBJECT_OFFSET_AS_ARY, + "Conflicting field offsets changed, rendering the rest of this test incorrect"); + + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let recv = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::LoadField { + recv, + id: FieldName::as_heap, + offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, + return_type: types::CPtr, + }); + let ivar = function.push_insn(entry, Insn::LoadField { + recv, + id: FieldName::Id(ID(1)), + offset: ROBJECT_OFFSET_AS_ARY as i32, + return_type: types::BasicObject, + }); + function.push_insn(entry, Insn::Return { val: ivar }); + function.seal_entries(); + + function.infer_types(); + function.optimize_load_store(); + + assert!( + function.validate().is_ok(), + "optimize_load_store aliased two loads with different return types: {:?}", + function.validate(), + ); + } + + #[test] + fn optimize_load_store_does_not_alias_loads_with_compatible_return_types() { + assert_eq!(ROBJECT_OFFSET_AS_HEAP_FIELDS, ROBJECT_OFFSET_AS_ARY, + "Conflicting field offsets changed, rendering the rest of this test incorrect"); + + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let recv = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(entry, Insn::LoadField { + recv, + id: FieldName::as_heap, + offset: ROBJECT_OFFSET_AS_HEAP_FIELDS as i32, + return_type: types::BasicObject, + }); + let ivar = function.push_insn(entry, Insn::LoadField { + recv, + id: FieldName::Id(ID(1)), + offset: ROBJECT_OFFSET_AS_ARY as i32, + return_type: types::Array, + }); + function.push_insn(entry, Insn::Return { val: ivar }); + function.seal_entries(); + + function.infer_types(); + function.optimize_load_store(); + + assert!( + function.validate().is_ok(), + "optimize_load_store failed to alias two loads with different, but compatible, return types: {:?}", + function.validate(), + ); + } } #[cfg(test)] @@ -3773,6 +9430,7 @@ mod infer_tests { fn test_const() { let mut function = Function::new(std::ptr::null()); let val = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qnil) }); + function.push_insn(function.entry_block, Insn::Unreachable); assert_bit_equal(function.infer_type(val), types::NilClass); } @@ -3782,6 +9440,8 @@ mod infer_tests { let mut function = Function::new(std::ptr::null()); let nil = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qnil) }); let val = function.push_insn(function.entry_block, Insn::Test { val: nil }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); function.infer_types(); assert_bit_equal(function.type_of(val), Type::from_cbool(false)); }); @@ -3793,6 +9453,8 @@ mod infer_tests { let mut function = Function::new(std::ptr::null()); let false_ = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qfalse) }); let val = function.push_insn(function.entry_block, Insn::Test { val: false_ }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); function.infer_types(); assert_bit_equal(function.type_of(val), Type::from_cbool(false)); }); @@ -3804,24 +9466,14 @@ mod infer_tests { let mut function = Function::new(std::ptr::null()); let true_ = function.push_insn(function.entry_block, Insn::Const { val: Const::Value(Qtrue) }); let val = function.push_insn(function.entry_block, Insn::Test { val: true_ }); + function.push_insn(function.entry_block, Insn::Unreachable); + function.seal_entries(); function.infer_types(); assert_bit_equal(function.type_of(val), Type::from_cbool(true)); }); } #[test] - fn test_unknown() { - crate::cruby::with_rubyvm(|| { - let mut function = Function::new(std::ptr::null()); - let param = function.push_insn(function.entry_block, Insn::Param { idx: SELF_PARAM_IDX }); - function.param_types.push(types::BasicObject); // self - let val = function.push_insn(function.entry_block, Insn::Test { val: param }); - function.infer_types(); - assert_bit_equal(function.type_of(val), types::CBool); - }); - } - - #[test] fn newarray() { let mut function = Function::new(std::ptr::null()); // Fake FrameState index of 0usize @@ -3847,14 +9499,66 @@ mod infer_tests { let v0 = function.push_insn(side, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(3)) }); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); - function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(4)) }); - function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![v1] })); - let param = function.push_insn(exit, Insn::Param { idx: 0 }); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![v1] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); + let param = function.push_insn(exit, Insn::Param); + function.push_insn(exit, Insn::Unreachable); + function.seal_entries(); crate::cruby::with_rubyvm(|| { function.infer_types(); }); - assert_bit_equal(function.type_of(param), types::Fixnum); + assert_bit_equal(function.type_of(param), Type::fixnum(3)); + } + + #[test] + fn self_loop_param_rotation_reaches_full_union() { + // bb_entry: jump bb_loop(c1, c2, c3, c4) // 4 distinct types + // bb_loop(p1, p2, p3, p4): + // jump bb_loop(p2, p3, p4, p1) // 4-cycle rotation + // + // Every param transitively flows into every other across enough trips + // around the loop, so the fixpoint for every param is the full union + // of all four input types. The fixpoint loop must not exit while a + // branch arm is still widening a param's type. + let mut function = Function::new(std::ptr::null()); + let entry = function.entry_block; + let loop_block = function.new_block(0); + + let c1 = function.push_insn(entry, Insn::Const { val: Const::Value(Qtrue) }); + let c2 = function.push_insn(entry, Insn::Const { val: Const::Value(Qfalse) }); + let c3 = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); + let c4 = function.push_insn(entry, Insn::Const { val: Const::Value(VALUE::fixnum_from_usize(7)) }); + function.push_insn(entry, Insn::Jump(BranchEdge { + target: loop_block, + args: vec![c1, c2, c3, c4], + })); + + let p1 = function.push_insn(loop_block, Insn::Param); + let p2 = function.push_insn(loop_block, Insn::Param); + let p3 = function.push_insn(loop_block, Insn::Param); + let p4 = function.push_insn(loop_block, Insn::Param); + function.push_insn(loop_block, Insn::Jump(BranchEdge { + target: loop_block, + args: vec![p2, p3, p4, p1], + })); + + function.seal_entries(); + crate::cruby::with_rubyvm(|| { + function.infer_types(); + }); + + let full = types::TrueClass + .union(types::FalseClass) + .union(types::NilClass) + .union(types::Fixnum); + assert_bit_equal(function.type_of(p1), full); + assert_bit_equal(function.type_of(p2), full); + assert_bit_equal(function.type_of(p3), full); + assert_bit_equal(function.type_of(p4), full); } #[test] @@ -3866,4373 +9570,18 @@ mod infer_tests { let v0 = function.push_insn(side, Insn::Const { val: Const::Value(Qtrue) }); function.push_insn(side, Insn::Jump(BranchEdge { target: exit, args: vec![v0] })); let val = function.push_insn(entry, Insn::Const { val: Const::CBool(false) }); - function.push_insn(entry, Insn::IfFalse { val, target: BranchEdge { target: side, args: vec![] } }); let v1 = function.push_insn(entry, Insn::Const { val: Const::Value(Qfalse) }); - function.push_insn(entry, Insn::Jump(BranchEdge { target: exit, args: vec![v1] })); - let param = function.push_insn(exit, Insn::Param { idx: 0 }); + function.push_insn(entry, Insn::CondBranch { + val, + if_true: BranchEdge { target: exit, args: vec![v1] }, + if_false: BranchEdge { target: side, args: vec![] }, + }); + let param = function.push_insn(exit, Insn::Param); + function.push_insn(exit, Insn::Unreachable); + function.seal_entries(); crate::cruby::with_rubyvm(|| { function.infer_types(); - assert_bit_equal(function.type_of(param), types::TrueClass.union(types::FalseClass)); + assert_bit_equal(function.type_of(param), types::TrueClass); }); } } - -#[cfg(test)] -mod tests { - use super::*; - use expect_test::{expect, Expect}; - - #[track_caller] - fn assert_method_hir(method: &str, hir: Expect) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); - unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; - let function = iseq_to_hir(iseq).unwrap(); - assert_function_hir(function, hir); - } - - fn iseq_contains_opcode(iseq: IseqPtr, expected_opcode: u32) -> bool { - let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; - let mut insn_idx = 0; - while insn_idx < iseq_size { - // Get the current pc and opcode - let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; - - // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. - let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } - .try_into() - .unwrap(); - if opcode == expected_opcode { - return true; - } - insn_idx += insn_len(opcode as usize); - } - false - } - - #[track_caller] - fn assert_method_hir_with_opcodes(method: &str, opcodes: &[u32], hir: Expect) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); - unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; - for &opcode in opcodes { - assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); - } - let function = iseq_to_hir(iseq).unwrap(); - assert_function_hir(function, hir); - } - - #[track_caller] - fn assert_method_hir_with_opcode(method: &str, opcode: u32, hir: Expect) { - assert_method_hir_with_opcodes(method, &[opcode], hir) - } - - #[track_caller] - pub fn assert_function_hir(function: Function, expected_hir: Expect) { - let actual_hir = format!("{}", FunctionPrinter::without_snapshot(&function)); - expected_hir.assert_eq(&actual_hir); - } - - #[track_caller] - pub fn assert_function_hir_with_frame_state(function: Function, expected_hir: Expect) { - let actual_hir = format!("{}", FunctionPrinter::with_snapshot(&function)); - expected_hir.assert_eq(&actual_hir); - } - - #[track_caller] - fn assert_compile_fails(method: &str, reason: ParseError) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); - unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; - let result = iseq_to_hir(iseq); - assert!(result.is_err(), "Expected an error but successfully compiled to HIR: {}", FunctionPrinter::without_snapshot(&result.unwrap())); - assert_eq!(result.unwrap_err(), reason); - } - - #[test] - fn test_cant_compile_optional() { - eval("def test(x=1) = 123"); - assert_compile_fails("test", ParseError::UnknownParameterType(ParameterType::Optional)); - } - - #[test] - fn test_putobject() { - eval("def test = 123"); - assert_method_hir_with_opcode("test", YARVINSN_putobject, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:Fixnum[123] = Const Value(123) - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_new_array() { - eval("def test = []"); - assert_method_hir_with_opcode("test", YARVINSN_newarray, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v3:ArrayExact = NewArray - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_new_array_with_element() { - eval("def test(a) = [a]"); - assert_method_hir_with_opcode("test", YARVINSN_newarray, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = NewArray v1 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_new_array_with_elements() { - eval("def test(a, b) = [a, b]"); - assert_method_hir_with_opcode("test", YARVINSN_newarray, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:ArrayExact = NewArray v1, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_new_range_inclusive_with_one_element() { - eval("def test(a) = (a..10)"); - assert_method_hir_with_opcode("test", YARVINSN_newrange, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[10] = Const Value(10) - v5:RangeExact = NewRange v1 NewRangeInclusive v3 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_new_range_inclusive_with_two_elements() { - eval("def test(a, b) = (a..b)"); - assert_method_hir_with_opcode("test", YARVINSN_newrange, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:RangeExact = NewRange v1 NewRangeInclusive v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_new_range_exclusive_with_one_element() { - eval("def test(a) = (a...10)"); - assert_method_hir_with_opcode("test", YARVINSN_newrange, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[10] = Const Value(10) - v5:RangeExact = NewRange v1 NewRangeExclusive v3 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_new_range_exclusive_with_two_elements() { - eval("def test(a, b) = (a...b)"); - assert_method_hir_with_opcode("test", YARVINSN_newrange, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:RangeExact = NewRange v1 NewRangeExclusive v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_array_dup() { - eval("def test = [1, 2, 3]"); - assert_method_hir_with_opcode("test", YARVINSN_duparray, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:ArrayExact = ArrayDup v2 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_hash_dup() { - eval("def test = {a: 1, b: 2}"); - assert_method_hir_with_opcode("test", YARVINSN_duphash, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:HashExact = HashDup v2 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_new_hash_empty() { - eval("def test = {}"); - assert_method_hir_with_opcode("test", YARVINSN_newhash, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v3:HashExact = NewHash - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_new_hash_with_elements() { - eval("def test(aval, bval) = {a: aval, b: bval}"); - assert_method_hir_with_opcode("test", YARVINSN_newhash, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v4:StaticSymbol[:a] = Const Value(VALUE(0x1000)) - v5:StaticSymbol[:b] = Const Value(VALUE(0x1008)) - v7:HashExact = NewHash v4: v1, v5: v2 - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn test_string_copy() { - eval("def test = \"hello\""); - assert_method_hir_with_opcode("test", YARVINSN_putchilledstring, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:StringExact = StringCopy v2 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_bignum() { - eval("def test = 999999999999999999999999999999999999"); - assert_method_hir_with_opcode("test", YARVINSN_putobject, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:Bignum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_flonum() { - eval("def test = 1.5"); - assert_method_hir_with_opcode("test", YARVINSN_putobject, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:Flonum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_heap_float() { - eval("def test = 1.7976931348623157e+308"); - assert_method_hir_with_opcode("test", YARVINSN_putobject, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:HeapFloat[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_static_sym() { - eval("def test = :foo"); - assert_method_hir_with_opcode("test", YARVINSN_putobject, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_opt_plus() { - eval("def test = 1+2"); - assert_method_hir_with_opcode("test", YARVINSN_opt_plus, expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - v5:BasicObject = SendWithoutBlock v2, :+, v3 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_hash_freeze() { - eval(" - def test = {}.freeze - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_hash_freeze, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:BasicObject = SendWithoutBlock v3, :freeze - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_opt_ary_freeze() { - eval(" - def test = [].freeze - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_ary_freeze, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:BasicObject = SendWithoutBlock v3, :freeze - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_opt_str_freeze() { - eval(" - def test = ''.freeze - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_str_freeze, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:BasicObject = SendWithoutBlock v3, :freeze - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_opt_str_uminus() { - eval(" - def test = -'' - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_str_uminus, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:BasicObject = SendWithoutBlock v3, :-@ - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_setlocal_getlocal() { - eval(" - def test - a = 1 - a - end - "); - assert_method_hir_with_opcodes("test", &[YARVINSN_getlocal_WC_0, YARVINSN_setlocal_WC_0], expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v3:Fixnum[1] = Const Value(1) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_nested_setlocal_getlocal() { - eval(" - l3 = 3 - _unused = _unused1 = nil - 1.times do |l2| - _ = nil - l2 = 2 - 1.times do |l1| - l1 = 1 - define_method(:test) do - l1 = l2 - l2 = l1 + l2 - l3 = l2 + l3 - end - end - end - "); - assert_method_hir_with_opcodes( - "test", - &[YARVINSN_getlocal_WC_1, YARVINSN_setlocal_WC_1, - YARVINSN_getlocal, YARVINSN_setlocal], - expect![[r#" - fn block (3 levels) in <compiled>@<compiled>:10: - bb0(v0:BasicObject): - v2:BasicObject = GetLocal l2, EP@4 - SetLocal l1, EP@3, v2 - v4:BasicObject = GetLocal l1, EP@3 - v5:BasicObject = GetLocal l2, EP@4 - v7:BasicObject = SendWithoutBlock v4, :+, v5 - SetLocal l2, EP@4, v7 - v9:BasicObject = GetLocal l2, EP@4 - v10:BasicObject = GetLocal l3, EP@5 - v12:BasicObject = SendWithoutBlock v9, :+, v10 - SetLocal l3, EP@5, v12 - CheckInterrupts - Return v12 - "#]] - ); - } - - #[test] - fn defined_ivar() { - eval(" - def test = defined?(@foo) - "); - assert_method_hir_with_opcode("test", YARVINSN_definedivar, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:BasicObject = DefinedIvar v0, :@foo - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn defined() { - eval(" - def test = return defined?(SeaChange), defined?(favourite), defined?($ruby) - "); - assert_method_hir_with_opcode("test", YARVINSN_defined, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:NilClass = Const Value(nil) - v4:StringExact|NilClass = Defined constant, v2 - v6:StringExact|NilClass = Defined func, v0 - v7:NilClass = Const Value(nil) - v9:StringExact|NilClass = Defined global-variable, v7 - v11:ArrayExact = NewArray v4, v6, v9 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_return_const() { - eval(" - def test(cond) - if cond - 3 - else - 4 - end - end - "); - assert_method_hir_with_opcode("test", YARVINSN_leave, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject): - CheckInterrupts - v5:CBool = Test v1 - IfFalse v5, bb1(v0, v1) - v7:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v7 - bb1(v11:BasicObject, v12:BasicObject): - v14:Fixnum[4] = Const Value(4) - CheckInterrupts - Return v14 - "#]]); - } - - #[test] - fn test_merge_const() { - eval(" - def test(cond) - if cond - result = 3 - else - result = 4 - end - result - end - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject): - v2:NilClass = Const Value(nil) - CheckInterrupts - v6:CBool = Test v1 - IfFalse v6, bb1(v0, v1, v2) - v8:Fixnum[3] = Const Value(3) - CheckInterrupts - Jump bb2(v0, v1, v8) - bb1(v12:BasicObject, v13:BasicObject, v14:NilClass): - v16:Fixnum[4] = Const Value(4) - Jump bb2(v12, v13, v16) - bb2(v18:BasicObject, v19:BasicObject, v20:Fixnum): - CheckInterrupts - Return v20 - "#]]); - } - - #[test] - fn test_opt_plus_fixnum() { - eval(" - def test(a, b) = a + b - test(1, 2); test(1, 2) - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :+, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_minus_fixnum() { - eval(" - def test(a, b) = a - b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_minus, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :-, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_mult_fixnum() { - eval(" - def test(a, b) = a * b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_mult, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :*, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_div_fixnum() { - eval(" - def test(a, b) = a / b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_div, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :/, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_mod_fixnum() { - eval(" - def test(a, b) = a % b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_mod, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :%, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_eq_fixnum() { - eval(" - def test(a, b) = a == b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_eq, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :==, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_neq_fixnum() { - eval(" - def test(a, b) = a != b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_neq, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :!=, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_lt_fixnum() { - eval(" - def test(a, b) = a < b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_lt, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :<, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_le_fixnum() { - eval(" - def test(a, b) = a <= b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_le, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :<=, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_opt_gt_fixnum() { - eval(" - def test(a, b) = a > b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_gt, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :>, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_loop() { - eval(" - def test - result = 0 - times = 10 - while times > 0 - result = result + 1 - times = times - 1 - end - result - end - test - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v2:NilClass = Const Value(nil) - v4:Fixnum[0] = Const Value(0) - v5:Fixnum[10] = Const Value(10) - CheckInterrupts - Jump bb2(v0, v4, v5) - bb2(v9:BasicObject, v10:BasicObject, v11:BasicObject): - v13:Fixnum[0] = Const Value(0) - v15:BasicObject = SendWithoutBlock v11, :>, v13 - CheckInterrupts - v18:CBool = Test v15 - IfTrue v18, bb1(v9, v10, v11) - v20:NilClass = Const Value(nil) - CheckInterrupts - Return v10 - bb1(v24:BasicObject, v25:BasicObject, v26:BasicObject): - v28:Fixnum[1] = Const Value(1) - v30:BasicObject = SendWithoutBlock v25, :+, v28 - v31:Fixnum[1] = Const Value(1) - v33:BasicObject = SendWithoutBlock v26, :-, v31 - Jump bb2(v24, v30, v33) - "#]]); - } - - #[test] - fn test_opt_ge_fixnum() { - eval(" - def test(a, b) = a >= b - test(1, 2); test(1, 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_ge, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :>=, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_display_types() { - eval(" - def test - cond = true - if cond - 3 - else - 4 - end - end - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v3:TrueClass = Const Value(true) - CheckInterrupts - v6:CBool[true] = Test v3 - IfFalse v6, bb1(v0, v3) - v8:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v8 - bb1(v12, v13): - v15 = Const Value(4) - CheckInterrupts - Return v15 - "#]]); - } - - #[test] - fn test_send_without_block() { - eval(" - def bar(a, b) - a+b - end - def test - bar(2, 3) - end - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_send_without_block, expect![[r#" - fn test@<compiled>:6: - bb0(v0:BasicObject): - v2:Fixnum[2] = Const Value(2) - v3:Fixnum[3] = Const Value(3) - v5:BasicObject = SendWithoutBlock v0, :bar, v2, v3 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_send_with_block() { - eval(" - def test(a) - a.each {|item| - item - } - end - test([1,2,3]) - "); - assert_method_hir_with_opcode("test", YARVINSN_send, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = Send v1, 0x1000, :each - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_intern_interpolated_symbol() { - eval(r#" - def test - :"foo#{123}" - end - "#); - assert_method_hir_with_opcode("test", YARVINSN_intern, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v3:Fixnum[123] = Const Value(123) - v5:BasicObject = ObjToString v3 - v7:String = AnyToString v3, str: v5 - v9:StringExact = StringConcat v2, v7 - v11:Symbol = StringIntern v9 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn different_objects_get_addresses() { - eval("def test = unknown_method([0], [1], '2', '2')"); - - // The 2 string literals have the same address because they're deduped. - assert_method_hir("test", expect![[r#" - fn test@<compiled>:1: - bb0(v0:BasicObject): - v2:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:ArrayExact = ArrayDup v2 - v5:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v7:ArrayExact = ArrayDup v5 - v8:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) - v10:StringExact = StringCopy v8 - v11:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) - v13:StringExact = StringCopy v11 - v15:BasicObject = SendWithoutBlock v0, :unknown_method, v4, v7, v10, v13 - CheckInterrupts - Return v15 - "#]]); - } - - #[test] - fn test_cant_compile_splat() { - eval(" - def test(a) = foo(*a) - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = ToArray v1 - SideExit UnknownCallType - "#]]); - } - - #[test] - fn test_cant_compile_block_arg() { - eval(" - def test(a) = foo(&a) - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - SideExit UnknownCallType - "#]]); - } - - #[test] - fn test_cant_compile_kwarg() { - eval(" - def test(a) = foo(a: 1) - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - SideExit UnknownCallType - "#]]); - } - - #[test] - fn test_cant_compile_kw_splat() { - eval(" - def test(a) = foo(**a) - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - SideExit UnknownCallType - "#]]); - } - - // TODO(max): Figure out how to generate a call with TAILCALL flag - - #[test] - fn test_cant_compile_super() { - eval(" - def test = super() - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - SideExit UnknownOpcode(invokesuper) - "#]]); - } - - #[test] - fn test_cant_compile_zsuper() { - eval(" - def test = super - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - SideExit UnknownOpcode(invokesuper) - "#]]); - } - - #[test] - fn test_cant_compile_super_forward() { - eval(" - def test(...) = super(...) - "); - assert_compile_fails("test", ParseError::UnknownParameterType(ParameterType::Forwardable)); - } - - #[test] - fn test_cant_compile_forwardable() { - eval("def forwardable(...) = nil"); - assert_compile_fails("forwardable", ParseError::UnknownParameterType(ParameterType::Forwardable)); - } - - // TODO(max): Figure out how to generate a call with OPT_SEND flag - - #[test] - fn test_cant_compile_kw_splat_mut() { - eval(" - def test(a) = foo **a, b: 1 - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) - v5:HashExact = NewHash - v7:BasicObject = SendWithoutBlock v3, :core#hash_merge_kwd, v5, v1 - v8:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) - v9:StaticSymbol[:b] = Const Value(VALUE(0x1008)) - v10:Fixnum[1] = Const Value(1) - v12:BasicObject = SendWithoutBlock v8, :core#hash_merge_ptr, v7, v9, v10 - SideExit UnknownCallType - "#]]); - } - - #[test] - fn test_cant_compile_splat_mut() { - eval(" - def test(*) = foo *, 1 - "); - assert_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:ArrayExact): - v4:ArrayExact = ToNewArray v1 - v5:Fixnum[1] = Const Value(1) - ArrayPush v4, v5 - SideExit UnknownCallType - "#]]); - } - - #[test] - fn test_cant_compile_forwarding() { - eval(" - def test(...) = foo(...) - "); - assert_compile_fails("test", ParseError::UnknownParameterType(ParameterType::Forwardable)); - } - - #[test] - fn test_opt_new() { - eval(" - class C; end - def test = C.new - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_new, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:BasicObject = GetConstantPath 0x1000 - v4:NilClass = Const Value(nil) - CheckInterrupts - Jump bb1(v0, v4, v3) - bb1(v8:BasicObject, v9:NilClass, v10:BasicObject): - v13:BasicObject = SendWithoutBlock v10, :new - Jump bb2(v8, v13, v9) - bb2(v15:BasicObject, v16:BasicObject, v17:NilClass): - CheckInterrupts - Return v16 - "#]]); - } - - #[test] - fn test_opt_newarray_send_max_no_elements() { - eval(" - def test = [].max - "); - // TODO(max): Rewrite to nil - assert_method_hir_with_opcode("test", YARVINSN_opt_newarray_send, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) - v4:BasicObject = ArrayMax - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_opt_newarray_send_max() { - eval(" - def test(a,b) = [a,b].max - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_newarray_send, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) - v6:BasicObject = ArrayMax v1, v2 - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_opt_newarray_send_min() { - eval(" - def test(a,b) - sum = a+b - result = [a,b].min - puts [1,2,3] - result - end - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_newarray_send, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v3:NilClass = Const Value(nil) - v4:NilClass = Const Value(nil) - v7:BasicObject = SendWithoutBlock v1, :+, v2 - SideExit UnknownNewarraySend(MIN) - "#]]); - } - - #[test] - fn test_opt_newarray_send_hash() { - eval(" - def test(a,b) - sum = a+b - result = [a,b].hash - puts [1,2,3] - result - end - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_newarray_send, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v3:NilClass = Const Value(nil) - v4:NilClass = Const Value(nil) - v7:BasicObject = SendWithoutBlock v1, :+, v2 - SideExit UnknownNewarraySend(HASH) - "#]]); - } - - #[test] - fn test_opt_newarray_send_pack() { - eval(" - def test(a,b) - sum = a+b - result = [a,b].pack 'C' - puts [1,2,3] - result - end - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_newarray_send, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v3:NilClass = Const Value(nil) - v4:NilClass = Const Value(nil) - v7:BasicObject = SendWithoutBlock v1, :+, v2 - v8:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v10:StringExact = StringCopy v8 - SideExit UnknownNewarraySend(PACK) - "#]]); - } - - // TODO(max): Add a test for VM_OPT_NEWARRAY_SEND_PACK_BUFFER - - #[test] - fn test_opt_newarray_send_include_p() { - eval(" - def test(a,b) - sum = a+b - result = [a,b].include? b - puts [1,2,3] - result - end - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_newarray_send, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v3:NilClass = Const Value(nil) - v4:NilClass = Const Value(nil) - v7:BasicObject = SendWithoutBlock v1, :+, v2 - SideExit UnknownNewarraySend(INCLUDE_P) - "#]]); - } - - #[test] - fn test_opt_length() { - eval(" - def test(a,b) = [a,b].length - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_length, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:ArrayExact = NewArray v1, v2 - v7:BasicObject = SendWithoutBlock v5, :length - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn test_opt_size() { - eval(" - def test(a,b) = [a,b].size - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_size, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:ArrayExact = NewArray v1, v2 - v7:BasicObject = SendWithoutBlock v5, :size - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn test_getinstancevariable() { - eval(" - def test = @foo - test - "); - assert_method_hir_with_opcode("test", YARVINSN_getinstancevariable, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:BasicObject = GetIvar v0, :@foo - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_setinstancevariable() { - eval(" - def test = @foo = 1 - test - "); - assert_method_hir_with_opcode("test", YARVINSN_setinstancevariable, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - SetIvar v0, :@foo, v2 - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_setglobal() { - eval(" - def test = $foo = 1 - test - "); - assert_method_hir_with_opcode("test", YARVINSN_setglobal, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - SetGlobal :$foo, v2 - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_getglobal() { - eval(" - def test = $foo - test - "); - assert_method_hir_with_opcode("test", YARVINSN_getglobal, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:BasicObject = GetGlobal :$foo - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_splatarray_mut() { - eval(" - def test(a) = [*a] - "); - assert_method_hir_with_opcode("test", YARVINSN_splatarray, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = ToNewArray v1 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_concattoarray() { - eval(" - def test(a) = [1, *a] - "); - assert_method_hir_with_opcode("test", YARVINSN_concattoarray, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - v5:ArrayExact = NewArray v3 - v7:ArrayExact = ToArray v1 - ArrayExtend v5, v7 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_pushtoarray_one_element() { - eval(" - def test(a) = [*a, 1] - "); - assert_method_hir_with_opcode("test", YARVINSN_pushtoarray, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = ToNewArray v1 - v5:Fixnum[1] = Const Value(1) - ArrayPush v4, v5 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_pushtoarray_multiple_elements() { - eval(" - def test(a) = [*a, 1, 2, 3] - "); - assert_method_hir_with_opcode("test", YARVINSN_pushtoarray, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v4:ArrayExact = ToNewArray v1 - v5:Fixnum[1] = Const Value(1) - v6:Fixnum[2] = Const Value(2) - v7:Fixnum[3] = Const Value(3) - ArrayPush v4, v5 - ArrayPush v4, v6 - ArrayPush v4, v7 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_aset() { - eval(" - def test(a, b) = a[b] = 1 - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_aset, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v4:NilClass = Const Value(nil) - v5:Fixnum[1] = Const Value(1) - v7:BasicObject = SendWithoutBlock v1, :[]=, v2, v5 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_aref() { - eval(" - def test(a, b) = a[b] - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_aref, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :[], v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_aref_with() { - eval(" - def test(a) = a['string lit triggers aref_with'] - "); - - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", "test")); - assert!(iseq_contains_opcode(iseq, YARVINSN_opt_aref_with)); - let function = iseq_to_hir(iseq).unwrap(); - assert_function_hir_with_frame_state(function, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v2:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [v1] } - v3:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v4:Any = Snapshot FrameState { pc: 0x1010, stack: [v1, v3], locals: [v1] } - v5:BasicObject = SendWithoutBlock v1, :[], v3 - v6:Any = Snapshot FrameState { pc: 0x1018, stack: [v5], locals: [v1] } - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn opt_empty_p() { - eval(" - def test(x) = x.empty? - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_empty_p, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v1, :empty? - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn opt_succ() { - eval(" - def test(x) = x.succ - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_succ, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v1, :succ - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn opt_and() { - eval(" - def test(x, y) = x & y - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_and, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :&, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn opt_or() { - eval(" - def test(x, y) = x | y - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_or, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :|, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn opt_not() { - eval(" - def test(x) = !x - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_not, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v4:BasicObject = SendWithoutBlock v1, :! - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn opt_regexpmatch2() { - eval(" - def test(regexp, matchee) = regexp =~ matchee - "); - assert_method_hir_with_opcode("test", YARVINSN_opt_regexpmatch2, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :=~, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - // Tests for ConstBase requires either constant or class definition, both - // of which can't be performed inside a method. - fn test_putspecialobject_vm_core_and_cbase() { - eval(" - def test - alias aliased __callee__ - end - "); - assert_method_hir_with_opcode("test", YARVINSN_putspecialobject, expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Class[VMFrozenCore] = Const Value(VALUE(0x1000)) - v3:BasicObject = PutSpecialObject CBase - v4:StaticSymbol[:aliased] = Const Value(VALUE(0x1008)) - v5:StaticSymbol[:__callee__] = Const Value(VALUE(0x1010)) - v7:BasicObject = SendWithoutBlock v2, :core#set_method_alias, v3, v4, v5 - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn opt_reverse() { - eval(" - def reverse_odd - a, b, c = @a, @b, @c - [a, b, c] - end - - def reverse_even - a, b, c, d = @a, @b, @c, @d - [a, b, c, d] - end - "); - assert_method_hir_with_opcode("reverse_odd", YARVINSN_opt_reverse, expect![[r#" - fn reverse_odd@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v2:NilClass = Const Value(nil) - v3:NilClass = Const Value(nil) - v6:BasicObject = GetIvar v0, :@a - v8:BasicObject = GetIvar v0, :@b - v10:BasicObject = GetIvar v0, :@c - v12:ArrayExact = NewArray v6, v8, v10 - CheckInterrupts - Return v12 - "#]]); - assert_method_hir_with_opcode("reverse_even", YARVINSN_opt_reverse, expect![[r#" - fn reverse_even@<compiled>:8: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v2:NilClass = Const Value(nil) - v3:NilClass = Const Value(nil) - v4:NilClass = Const Value(nil) - v7:BasicObject = GetIvar v0, :@a - v9:BasicObject = GetIvar v0, :@b - v11:BasicObject = GetIvar v0, :@c - v13:BasicObject = GetIvar v0, :@d - v15:ArrayExact = NewArray v7, v9, v11, v13 - CheckInterrupts - Return v15 - "#]]); - } - - #[test] - fn test_branchnil() { - eval(" - def test(x) = x&.itself - "); - assert_method_hir_with_opcode("test", YARVINSN_branchnil, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - CheckInterrupts - v5:CBool = IsNil v1 - IfTrue v5, bb1(v0, v1, v1) - v8:BasicObject = SendWithoutBlock v1, :itself - Jump bb1(v0, v1, v8) - bb1(v10:BasicObject, v11:BasicObject, v12:BasicObject): - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_invokebuiltin_delegate_annotated() { - assert_method_hir_with_opcode("Float", YARVINSN_opt_invokebuiltin_delegate_leave, expect![[r#" - fn Float@<internal:kernel>: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject, v3:BasicObject): - v6:Float = InvokeBuiltin rb_f_float, v0, v1, v2 - Jump bb1(v0, v1, v2, v3, v6) - bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject, v12:Float): - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_invokebuiltin_cexpr_annotated() { - assert_method_hir_with_opcode("class", YARVINSN_opt_invokebuiltin_delegate_leave, expect![[r#" - fn class@<internal:kernel>: - bb0(v0:BasicObject): - v3:Class = InvokeBuiltin _bi20, v0 - Jump bb1(v0, v3) - bb1(v5:BasicObject, v6:Class): - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_invokebuiltin_delegate_with_args() { - // Using an unannotated builtin to test InvokeBuiltin generation - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Dir", "open")); - assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate), "iseq Dir.open does not contain invokebuiltin"); - let function = iseq_to_hir(iseq).unwrap(); - assert_function_hir(function, expect![[r#" - fn open@<internal:dir>: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject, v3:BasicObject, v4:BasicObject): - v5:NilClass = Const Value(nil) - v8:BasicObject = InvokeBuiltin dir_s_open, v0, v1, v2 - SideExit UnknownOpcode(getblockparamproxy) - "#]]); - } - - #[test] - fn test_invokebuiltin_delegate_without_args() { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "enable")); - assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate_leave), "iseq GC.enable does not contain invokebuiltin"); - let function = iseq_to_hir(iseq).unwrap(); - assert_function_hir(function, expect![[r#" - fn enable@<internal:gc>: - bb0(v0:BasicObject): - v3:BasicObject = InvokeBuiltin gc_enable, v0 - Jump bb1(v0, v3) - bb1(v5:BasicObject, v6:BasicObject): - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_invokebuiltin_with_args() { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "start")); - assert!(iseq_contains_opcode(iseq, YARVINSN_invokebuiltin), "iseq GC.start does not contain invokebuiltin"); - let function = iseq_to_hir(iseq).unwrap(); - assert_function_hir(function, expect![[r#" - fn start@<internal:gc>: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject, v3:BasicObject, v4:BasicObject): - v6:FalseClass = Const Value(false) - v8:BasicObject = InvokeBuiltin gc_start_internal, v0, v1, v2, v3, v6 - CheckInterrupts - Return v8 - "#]]); - } - - #[test] - fn dupn() { - eval(" - def test(x) = (x[0, 1] ||= 2) - "); - assert_method_hir_with_opcode("test", YARVINSN_dupn, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:NilClass = Const Value(nil) - v4:Fixnum[0] = Const Value(0) - v5:Fixnum[1] = Const Value(1) - v7:BasicObject = SendWithoutBlock v1, :[], v4, v5 - CheckInterrupts - v10:CBool = Test v7 - IfTrue v10, bb1(v0, v1, v3, v1, v4, v5, v7) - v12:Fixnum[2] = Const Value(2) - v14:BasicObject = SendWithoutBlock v1, :[]=, v4, v5, v12 - CheckInterrupts - Return v12 - bb1(v18:BasicObject, v19:BasicObject, v20:NilClass, v21:BasicObject, v22:Fixnum[0], v23:Fixnum[1], v24:BasicObject): - CheckInterrupts - Return v24 - "#]]); - } - - #[test] - fn test_objtostring_anytostring() { - eval(" - def test = \"#{1}\" - "); - assert_method_hir_with_opcode("test", YARVINSN_objtostring, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v3:Fixnum[1] = Const Value(1) - v5:BasicObject = ObjToString v3 - v7:String = AnyToString v3, str: v5 - v9:StringExact = StringConcat v2, v7 - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_string_concat() { - eval(r##" - def test = "#{1}#{2}#{3}" - "##); - assert_method_hir_with_opcode("test", YARVINSN_concatstrings, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v4:BasicObject = ObjToString v2 - v6:String = AnyToString v2, str: v4 - v7:Fixnum[2] = Const Value(2) - v9:BasicObject = ObjToString v7 - v11:String = AnyToString v7, str: v9 - v12:Fixnum[3] = Const Value(3) - v14:BasicObject = ObjToString v12 - v16:String = AnyToString v12, str: v14 - v18:StringExact = StringConcat v6, v11, v16 - CheckInterrupts - Return v18 - "#]]); - } - - #[test] - fn test_string_concat_empty() { - eval(r##" - def test = "#{}" - "##); - assert_method_hir_with_opcode("test", YARVINSN_concatstrings, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v3:NilClass = Const Value(nil) - v5:BasicObject = ObjToString v3 - v7:String = AnyToString v3, str: v5 - v9:StringExact = StringConcat v2, v7 - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_toregexp() { - eval(r##" - def test = /#{1}#{2}#{3}/ - "##); - assert_method_hir_with_opcode("test", YARVINSN_toregexp, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v4:BasicObject = ObjToString v2 - v6:String = AnyToString v2, str: v4 - v7:Fixnum[2] = Const Value(2) - v9:BasicObject = ObjToString v7 - v11:String = AnyToString v7, str: v9 - v12:Fixnum[3] = Const Value(3) - v14:BasicObject = ObjToString v12 - v16:String = AnyToString v12, str: v14 - v18:RegexpExact = ToRegexp v6, v11, v16 - CheckInterrupts - Return v18 - "#]]); - } - - #[test] - fn test_toregexp_with_options() { - eval(r##" - def test = /#{1}#{2}/mixn - "##); - assert_method_hir_with_opcode("test", YARVINSN_toregexp, expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v4:BasicObject = ObjToString v2 - v6:String = AnyToString v2, str: v4 - v7:Fixnum[2] = Const Value(2) - v9:BasicObject = ObjToString v7 - v11:String = AnyToString v7, str: v9 - v13:RegexpExact = ToRegexp v6, v11, MULTILINE|IGNORECASE|EXTENDED|NOENCODING - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn throw() { - eval(" - define_method(:throw_return) { return 1 } - define_method(:throw_break) { break 2 } - "); - assert_method_hir_with_opcode("throw_return", YARVINSN_throw, expect![[r#" - fn block in <compiled>@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - Throw TAG_RETURN, v2 - "#]]); - assert_method_hir_with_opcode("throw_break", YARVINSN_throw, expect![[r#" - fn block in <compiled>@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[2] = Const Value(2) - Throw TAG_BREAK, v2 - "#]]); - } -} - -#[cfg(test)] -mod graphviz_tests { - use super::*; - use expect_test::{expect, Expect}; - - #[track_caller] - fn assert_optimized_graphviz(method: &str, expected: Expect) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); - unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; - let mut function = iseq_to_hir(iseq).unwrap(); - function.optimize(); - function.validate().unwrap(); - let actual = format!("{}", FunctionGraphvizPrinter::new(&function)); - expected.assert_eq(&actual); - } - - #[test] - fn test_guard_fixnum_or_fixnum() { - eval(r#" - def test(x, y) = x | y - - test(1, 2) - "#); - assert_optimized_graphviz("test", expect![[r#" - digraph G { # test@<compiled>:2 - node [shape=plaintext]; - mode=hier; overlap=false; splines=true; - bb0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> - <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject) </TD></TR> - <TR><TD ALIGN="left" PORT="v9">PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, 29) </TD></TR> - <TR><TD ALIGN="left" PORT="v10">v10:Fixnum = GuardType v1, Fixnum </TD></TR> - <TR><TD ALIGN="left" PORT="v11">v11:Fixnum = GuardType v2, Fixnum </TD></TR> - <TR><TD ALIGN="left" PORT="v12">v12:Fixnum = FixnumOr v10, v11 </TD></TR> - <TR><TD ALIGN="left" PORT="v7">CheckInterrupts </TD></TR> - <TR><TD ALIGN="left" PORT="v8">Return v12 </TD></TR> - </TABLE>>]; - } - "#]]); - } - - #[test] - fn test_multiple_blocks() { - eval(r#" - def test(c) - if c - 3 - else - 4 - end - end - - test(1) - test("x") - "#); - assert_optimized_graphviz("test", expect![[r#" - digraph G { # test@<compiled>:3 - node [shape=plaintext]; - mode=hier; overlap=false; splines=true; - bb0 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> - <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb0(v0:BasicObject, v1:BasicObject) </TD></TR> - <TR><TD ALIGN="left" PORT="v4">CheckInterrupts </TD></TR> - <TR><TD ALIGN="left" PORT="v5">v5:CBool = Test v1 </TD></TR> - <TR><TD ALIGN="left" PORT="v6">IfFalse v5, bb1(v0, v1) </TD></TR> - <TR><TD ALIGN="left" PORT="v7">v7:Fixnum[3] = Const Value(3) </TD></TR> - <TR><TD ALIGN="left" PORT="v9">CheckInterrupts </TD></TR> - <TR><TD ALIGN="left" PORT="v10">Return v7 </TD></TR> - </TABLE>>]; - bb0:v6 -> bb1:params; - bb1 [label=<<TABLE BORDER="0" CELLBORDER="1" CELLSPACING="0"> - <TR><TD ALIGN="LEFT" PORT="params" BGCOLOR="gray">bb1(v11:BasicObject, v12:BasicObject) </TD></TR> - <TR><TD ALIGN="left" PORT="v14">v14:Fixnum[4] = Const Value(4) </TD></TR> - <TR><TD ALIGN="left" PORT="v16">CheckInterrupts </TD></TR> - <TR><TD ALIGN="left" PORT="v17">Return v14 </TD></TR> - </TABLE>>]; - } - "#]]); - } -} - -#[cfg(test)] -mod opt_tests { - use super::*; - use super::tests::assert_function_hir; - use expect_test::{expect, Expect}; - - #[track_caller] - fn assert_optimized_method_hir(method: &str, hir: Expect) { - let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); - unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; - let mut function = iseq_to_hir(iseq).unwrap(); - function.optimize(); - function.validate().unwrap(); - assert_function_hir(function, hir); - } - - #[test] - fn test_fold_iftrue_away() { - eval(" - def test - cond = true - if cond - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:TrueClass = Const Value(true) - CheckInterrupts - v8:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v8 - "#]]); - } - - #[test] - fn test_fold_iftrue_into_jump() { - eval(" - def test - cond = false - if cond - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:FalseClass = Const Value(false) - CheckInterrupts - v15:Fixnum[4] = Const Value(4) - CheckInterrupts - Return v15 - "#]]); - } - - #[test] - fn test_fold_fixnum_add() { - eval(" - def test - 1 + 2 + 3 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v16:Fixnum[3] = Const Value(3) - v6:Fixnum[3] = Const Value(3) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v17:Fixnum[6] = Const Value(6) - CheckInterrupts - Return v17 - "#]]); - } - - #[test] - fn test_fold_fixnum_sub() { - eval(" - def test - 5 - 3 - 1 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[5] = Const Value(5) - v3:Fixnum[3] = Const Value(3) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v16:Fixnum[2] = Const Value(2) - v6:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v17:Fixnum[1] = Const Value(1) - CheckInterrupts - Return v17 - "#]]); - } - - #[test] - fn test_fold_fixnum_sub_large_negative_result() { - eval(" - def test - 0 - 1073741825 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[0] = Const Value(0) - v3:Fixnum[1073741825] = Const Value(1073741825) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v11:Fixnum[-1073741825] = Const Value(-1073741825) - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_fold_fixnum_mult() { - eval(" - def test - 6 * 7 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[6] = Const Value(6) - v3:Fixnum[7] = Const Value(7) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v11:Fixnum[42] = Const Value(42) - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_fold_fixnum_mult_zero() { - eval(" - def test(n) - 0 * n + n * 0 - end - test 1; test 2 - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[0] = Const Value(0) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v15:Fixnum = GuardType v1, Fixnum - v22:Fixnum[0] = Const Value(0) - v6:Fixnum[0] = Const Value(0) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v18:Fixnum = GuardType v1, Fixnum - v23:Fixnum[0] = Const Value(0) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v24:Fixnum[0] = Const Value(0) - CheckInterrupts - Return v24 - "#]]); - } - - #[test] - fn test_fold_fixnum_less() { - eval(" - def test - if 1 < 2 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v22:TrueClass = Const Value(true) - CheckInterrupts - v10:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_fold_fixnum_less_equal() { - eval(" - def test - if 1 <= 2 && 2 <= 2 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LE) - v32:TrueClass = Const Value(true) - CheckInterrupts - v10:Fixnum[2] = Const Value(2) - v11:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LE) - v34:TrueClass = Const Value(true) - CheckInterrupts - v18:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v18 - "#]]); - } - - #[test] - fn test_fold_fixnum_greater() { - eval(" - def test - if 2 > 1 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[2] = Const Value(2) - v3:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GT) - v22:TrueClass = Const Value(true) - CheckInterrupts - v10:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_fold_fixnum_greater_equal() { - eval(" - def test - if 2 >= 1 && 2 >= 2 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[2] = Const Value(2) - v3:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GE) - v32:TrueClass = Const Value(true) - CheckInterrupts - v10:Fixnum[2] = Const Value(2) - v11:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GE) - v34:TrueClass = Const Value(true) - CheckInterrupts - v18:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v18 - "#]]); - } - - #[test] - fn test_fold_fixnum_eq_false() { - eval(" - def test - if 1 == 2 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v22:FalseClass = Const Value(false) - CheckInterrupts - v16:Fixnum[4] = Const Value(4) - CheckInterrupts - Return v16 - "#]]); - } - - #[test] - fn test_fold_fixnum_eq_true() { - eval(" - def test - if 2 == 2 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[2] = Const Value(2) - v3:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v22:TrueClass = Const Value(true) - CheckInterrupts - v10:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_fold_fixnum_neq_true() { - eval(" - def test - if 1 != 2 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_NEQ) - v23:TrueClass = Const Value(true) - CheckInterrupts - v10:Fixnum[3] = Const Value(3) - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_fold_fixnum_neq_false() { - eval(" - def test - if 2 != 2 - 3 - else - 4 - end - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[2] = Const Value(2) - v3:Fixnum[2] = Const Value(2) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_NEQ) - v23:FalseClass = Const Value(false) - CheckInterrupts - v16:Fixnum[4] = Const Value(4) - CheckInterrupts - Return v16 - "#]]); - } - - #[test] - fn test_replace_guard_if_known_fixnum() { - eval(" - def test(a) - a + 1 - end - test(2); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v10:Fixnum = GuardType v1, Fixnum - v11:Fixnum = FixnumAdd v10, v3 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_param_forms_get_bb_param() { - eval(" - def rest(*array) = array - def kw(k:) = k - def kw_rest(**k) = k - def post(*rest, post) = post - def block(&b) = nil - "); - - assert_optimized_method_hir("rest", expect![[r#" - fn rest@<compiled>:2: - bb0(v0:BasicObject, v1:ArrayExact): - CheckInterrupts - Return v1 - "#]]); - // extra hidden param for the set of specified keywords - assert_optimized_method_hir("kw", expect![[r#" - fn kw@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - CheckInterrupts - Return v1 - "#]]); - assert_optimized_method_hir("kw_rest", expect![[r#" - fn kw_rest@<compiled>:4: - bb0(v0:BasicObject, v1:BasicObject): - CheckInterrupts - Return v1 - "#]]); - assert_optimized_method_hir("block", expect![[r#" - fn block@<compiled>:6: - bb0(v0:BasicObject, v1:BasicObject): - v3:NilClass = Const Value(nil) - CheckInterrupts - Return v3 - "#]]); - assert_optimized_method_hir("post", expect![[r#" - fn post@<compiled>:5: - bb0(v0:BasicObject, v1:ArrayExact, v2:BasicObject): - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_optimize_top_level_call_into_send_direct() { - eval(" - def foo - end - def test - foo - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:5: - bb0(v0:BasicObject): - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v8:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] - v9:BasicObject = SendWithoutBlockDirect v8, :foo (0x1038) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_optimize_nonexistent_top_level_call() { - eval(" - def foo - end - def test - foo - end - test; test - undef :foo - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:5: - bb0(v0:BasicObject): - v3:BasicObject = SendWithoutBlock v0, :foo - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_optimize_private_top_level_call() { - eval(" - def foo - end - private :foo - def test - foo - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:6: - bb0(v0:BasicObject): - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v8:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] - v9:BasicObject = SendWithoutBlockDirect v8, :foo (0x1038) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_optimize_top_level_call_with_overloaded_cme() { - eval(" - def test - Integer(3) - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[3] = Const Value(3) - PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) - v9:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] - v10:BasicObject = SendWithoutBlockDirect v9, :Integer (0x1038), v2 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_optimize_top_level_call_with_args_into_send_direct() { - eval(" - def foo a, b - end - def test - foo 1, 2 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:5: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v10:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] - v11:BasicObject = SendWithoutBlockDirect v10, :foo (0x1038), v2, v3 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_optimize_top_level_sends_into_send_direct() { - eval(" - def foo - end - def bar - end - def test - foo - bar - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:7: - bb0(v0:BasicObject): - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v10:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] - v11:BasicObject = SendWithoutBlockDirect v10, :foo (0x1038) - PatchPoint MethodRedefined(Object@0x1000, bar@0x1040, cme:0x1048) - v13:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] - v14:BasicObject = SendWithoutBlockDirect v13, :bar (0x1038) - CheckInterrupts - Return v14 - "#]]); - } - - #[test] - fn test_dont_optimize_fixnum_add_if_redefined() { - eval(" - class Integer - def +(other) - 100 - end - end - def test(a, b) = a + b - test(1,2); test(3,4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:7: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:BasicObject = SendWithoutBlock v1, :+, v2 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_add_both_profiled() { - eval(" - def test(a, b) = a + b - test(1,2); test(3,4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v10:Fixnum = GuardType v1, Fixnum - v11:Fixnum = GuardType v2, Fixnum - v12:Fixnum = FixnumAdd v10, v11 - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_add_left_profiled() { - eval(" - def test(a) = a + 1 - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v10:Fixnum = GuardType v1, Fixnum - v11:Fixnum = FixnumAdd v10, v3 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_add_right_profiled() { - eval(" - def test(a) = 1 + a - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v10:Fixnum = GuardType v1, Fixnum - v11:Fixnum = FixnumAdd v3, v10 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_lt_both_profiled() { - eval(" - def test(a, b) = a < b - test(1,2); test(3,4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v10:Fixnum = GuardType v1, Fixnum - v11:Fixnum = GuardType v2, Fixnum - v12:BoolExact = FixnumLt v10, v11 - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_lt_left_profiled() { - eval(" - def test(a) = a < 1 - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v10:Fixnum = GuardType v1, Fixnum - v11:BoolExact = FixnumLt v10, v3 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_optimize_send_into_fixnum_lt_right_profiled() { - eval(" - def test(a) = 1 < a - test(1); test(3) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v10:Fixnum = GuardType v1, Fixnum - v11:BoolExact = FixnumLt v3, v10 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_eliminate_new_array() { - eval(" - def test() - c = [] - 5 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v4:ArrayExact = NewArray - v5:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_eliminate_new_range() { - eval(" - def test() - c = (1..2) - 5 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:RangeExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_do_not_eliminate_new_range_non_fixnum() { - eval(" - def test() - _ = (-'a'..'b') - 0 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v4:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) - v6:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v8:StringExact = StringCopy v6 - v10:RangeExact = NewRange v4 NewRangeInclusive v8 - v11:Fixnum[0] = Const Value(0) - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_eliminate_new_array_with_elements() { - eval(" - def test(a) - c = [a] - 5 - end - test(1); test(2) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject): - v2:NilClass = Const Value(nil) - v5:ArrayExact = NewArray v1 - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_new_hash() { - eval(" - def test() - c = {} - 5 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v4:HashExact = NewHash - v5:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_no_eliminate_new_hash_with_elements() { - eval(" - def test(aval, bval) - c = {a: aval, b: bval} - 5 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v3:NilClass = Const Value(nil) - v5:StaticSymbol[:a] = Const Value(VALUE(0x1000)) - v6:StaticSymbol[:b] = Const Value(VALUE(0x1008)) - v8:HashExact = NewHash v5: v1, v6: v2 - v9:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_eliminate_array_dup() { - eval(" - def test - c = [1, 2] - 5 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v5:ArrayExact = ArrayDup v3 - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_hash_dup() { - eval(" - def test - c = {a: 1, b: 2} - 5 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v3:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v5:HashExact = HashDup v3 - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_putself() { - eval(" - def test() - c = self - 5 - end - test; test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_eliminate_string_copy() { - eval(r#" - def test() - c = "abc" - 5 - end - test; test - "#); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v3:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v5:StringExact = StringCopy v3 - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_add() { - eval(" - def test(a, b) - a + b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_PLUS) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_sub() { - eval(" - def test(a, b) - a - b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MINUS) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_mul() { - eval(" - def test(a, b) - a * b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MULT) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_do_not_eliminate_fixnum_div() { - eval(" - def test(a, b) - a / b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_DIV) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v13:Fixnum = FixnumDiv v11, v12 - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_do_not_eliminate_fixnum_mod() { - eval(" - def test(a, b) - a % b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_MOD) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v13:Fixnum = FixnumMod v11, v12 - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_lt() { - eval(" - def test(a, b) - a < b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LT) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_le() { - eval(" - def test(a, b) - a <= b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_LE) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_gt() { - eval(" - def test(a, b) - a > b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GT) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_ge() { - eval(" - def test(a, b) - a >= b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_GE) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_eq() { - eval(" - def test(a, b) - a == b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - v11:Fixnum = GuardType v1, Fixnum - v12:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_eliminate_fixnum_neq() { - eval(" - def test(a, b) - a != b - 5 - end - test(1, 2); test(3, 4) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_NEQ) - v12:Fixnum = GuardType v1, Fixnum - v13:Fixnum = GuardType v2, Fixnum - v6:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_do_not_eliminate_get_constant_path() { - eval(" - def test() - C - 5 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:BasicObject = GetConstantPath 0x1000 - v4:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn kernel_itself_const() { - eval(" - def test(x) = x.itself - test(0) # profile - test(1) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) - v9:Fixnum = GuardType v1, Fixnum - v10:BasicObject = CCall itself@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn kernel_itself_known_type() { - eval(" - def test = [].itself - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact = NewArray - PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) - v10:BasicObject = CCall itself@0x1038, v3 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn eliminate_kernel_itself() { - eval(" - def test - x = [].itself - 1 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v4:ArrayExact = NewArray - PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) - v12:BasicObject = CCall itself@0x1038, v4 - v7:Fixnum[1] = Const Value(1) - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn eliminate_module_name() { - eval(" - module M; end - def test - x = M.name - 1 - end - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:4: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, M) - v13:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - PatchPoint MethodRedefined(Module@0x1010, name@0x1018, cme:0x1020) - v15:StringExact|NilClass = CCall name@0x1048, v13 - v7:Fixnum[1] = Const Value(1) - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn eliminate_array_length() { - eval(" - def test - x = [].length - 5 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v4:ArrayExact = NewArray - PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) - v12:Fixnum = CCall length@0x1038, v4 - v7:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn normal_class_type_inference() { - eval(" - class C; end - def test = C - test # Warm the constant cache - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, C) - v9:Class[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn core_classes_type_inference() { - eval(" - def test = [String, Class, Module, BasicObject] - test # Warm the constant cache - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, String) - v17:Class[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1010, Class) - v20:Class[VALUE(0x1018)] = Const Value(VALUE(0x1018)) - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1020, Module) - v23:Class[VALUE(0x1028)] = Const Value(VALUE(0x1028)) - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1030, BasicObject) - v26:Class[VALUE(0x1038)] = Const Value(VALUE(0x1038)) - v11:ArrayExact = NewArray v17, v20, v23, v26 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn module_instances_are_module_exact() { - eval(" - def test = [Enumerable, Kernel] - test # Warm the constant cache - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, Enumerable) - v13:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1010, Kernel) - v16:ModuleExact[VALUE(0x1018)] = Const Value(VALUE(0x1018)) - v7:ArrayExact = NewArray v13, v16 - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn module_subclasses_are_not_module_exact() { - eval(" - class ModuleSubclass < Module; end - MY_MODULE = ModuleSubclass.new - def test = MY_MODULE - test # Warm the constant cache - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:4: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, MY_MODULE) - v9:BasicObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn eliminate_array_size() { - eval(" - def test - x = [].size - 5 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v1:NilClass = Const Value(nil) - v4:ArrayExact = NewArray - PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) - v12:Fixnum = CCall size@0x1038, v4 - v7:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn kernel_itself_argc_mismatch() { - eval(" - def test = 1.itself(0) - test rescue 0 - test rescue 0 - "); - // Not specialized - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v3:Fixnum[0] = Const Value(0) - v5:BasicObject = SendWithoutBlock v2, :itself, v3 - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn const_send_direct_integer() { - eval(" - def test(x) = 1.zero? - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - v3:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1000, zero?@0x1008, cme:0x1010) - v10:BasicObject = SendWithoutBlockDirect v3, :zero? (0x1038) - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn class_known_send_direct_array() { - eval(" - def test(x) - a = [1,2,3] - a.first - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject, v1:BasicObject): - v2:NilClass = Const Value(nil) - v4:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v6:ArrayExact = ArrayDup v4 - PatchPoint MethodRedefined(Array@0x1008, first@0x1010, cme:0x1018) - v13:BasicObject = SendWithoutBlockDirect v6, :first (0x1040) - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn send_direct_to_module() { - eval(" - module M; end - def test = M.class - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, M) - v11:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - PatchPoint MethodRedefined(Module@0x1010, class@0x1018, cme:0x1020) - v13:BasicObject = SendWithoutBlockDirect v11, :class (0x1048) - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_send_direct_to_instance_method() { - eval(" - class C - def foo - 3 - end - end - - def test(c) = c.foo - c = C.new - test c - test c - "); - - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:8: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) - v9:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C] - v10:BasicObject = SendWithoutBlockDirect v9, :foo (0x1038) - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn dont_specialize_call_to_iseq_with_opt() { - eval(" - def foo(arg=1) = 1 - def test = foo 1 - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v4:BasicObject = SendWithoutBlock v0, :foo, v2 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn dont_specialize_call_to_iseq_with_block() { - eval(" - def foo(&block) = 1 - def test = foo {|| } - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:BasicObject = Send v0, 0x1000, :foo - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn dont_specialize_call_to_iseq_with_rest() { - eval(" - def foo(*args) = 1 - def test = foo 1 - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - v4:BasicObject = SendWithoutBlock v0, :foo, v2 - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn dont_specialize_call_to_iseq_with_kw() { - eval(" - def foo(a:) = 1 - def test = foo(a: 1) - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - SideExit UnknownCallType - "#]]); - } - - #[test] - fn dont_specialize_call_to_iseq_with_kwrest() { - eval(" - def foo(**args) = 1 - def test = foo(a: 1) - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - SideExit UnknownCallType - "#]]); - } - - #[test] - fn string_bytesize_simple() { - eval(" - def test = 'abc'.bytesize - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:StringExact = StringCopy v2 - PatchPoint MethodRedefined(String@0x1008, bytesize@0x1010, cme:0x1018) - v11:Fixnum = CCall bytesize@0x1040, v4 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn dont_replace_get_constant_path_with_empty_ic() { - eval(" - def test = Kernel - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:BasicObject = GetConstantPath 0x1000 - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn dont_replace_get_constant_path_with_invalidated_ic() { - eval(" - def test = Kernel - test - Kernel = 5 - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:BasicObject = GetConstantPath 0x1000 - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn replace_get_constant_path_with_const() { - eval(" - def test = Kernel - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, Kernel) - v9:ModuleExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn replace_nested_get_constant_path_with_const() { - eval(" - module Foo - module Bar - class C - end - end - end - def test = Foo::Bar::C - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:8: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, Foo::Bar::C) - v9:Class[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_opt_new_no_initialize() { - eval(" - class C; end - def test = C.new - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, C) - v24:Class[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v4:NilClass = Const Value(nil) - CheckInterrupts - v13:BasicObject = SendWithoutBlock v24, :new - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_opt_new_initialize() { - eval(" - class C - def initialize x - @x = x - end - end - def test = C.new 1 - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:7: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, C) - v26:Class[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v4:NilClass = Const Value(nil) - v5:Fixnum[1] = Const Value(1) - CheckInterrupts - v15:BasicObject = SendWithoutBlock v26, :new, v5 - CheckInterrupts - Return v15 - "#]]); - } - - #[test] - fn test_opt_length() { - eval(" - def test(a,b) = [a,b].length - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:ArrayExact = NewArray v1, v2 - PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) - v12:Fixnum = CCall length@0x1038, v5 - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_opt_size() { - eval(" - def test(a,b) = [a,b].size - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - v5:ArrayExact = NewArray v1, v2 - PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) - v12:Fixnum = CCall size@0x1038, v5 - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_getinstancevariable() { - eval(" - def test = @foo - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:BasicObject = GetIvar v0, :@foo - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_setinstancevariable() { - eval(" - def test = @foo = 1 - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - SetIvar v0, :@foo, v2 - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_elide_freeze_with_frozen_hash() { - eval(" - def test = {}.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_dont_optimize_hash_freeze_if_redefined() { - eval(" - class Hash - def freeze; end - end - def test = {}.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:5: - bb0(v0:BasicObject): - v3:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:BasicObject = SendWithoutBlock v3, :freeze - CheckInterrupts - Return v4 - "#]]); - } - - #[test] - fn test_elide_freeze_with_refrozen_hash() { - eval(" - def test = {}.freeze.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) - PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_no_elide_freeze_with_unfrozen_hash() { - eval(" - def test = {}.dup.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:HashExact = NewHash - v5:BasicObject = SendWithoutBlock v3, :dup - v7:BasicObject = SendWithoutBlock v5, :freeze - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn test_no_elide_freeze_hash_with_args() { - eval(" - def test = {}.freeze(nil) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:HashExact = NewHash - v4:NilClass = Const Value(nil) - v6:BasicObject = SendWithoutBlock v3, :freeze, v4 - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_elide_freeze_with_frozen_ary() { - eval(" - def test = [].freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_elide_freeze_with_refrozen_ary() { - eval(" - def test = [].freeze.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_no_elide_freeze_with_unfrozen_ary() { - eval(" - def test = [].dup.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact = NewArray - v5:BasicObject = SendWithoutBlock v3, :dup - v7:BasicObject = SendWithoutBlock v5, :freeze - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn test_no_elide_freeze_ary_with_args() { - eval(" - def test = [].freeze(nil) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact = NewArray - v4:NilClass = Const Value(nil) - v6:BasicObject = SendWithoutBlock v3, :freeze, v4 - CheckInterrupts - Return v6 - "#]]); - } - - #[test] - fn test_elide_freeze_with_frozen_str() { - eval(" - def test = ''.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_elide_freeze_with_refrozen_str() { - eval(" - def test = ''.freeze.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) - PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_no_elide_freeze_with_unfrozen_str() { - eval(" - def test = ''.dup.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:StringExact = StringCopy v2 - v6:BasicObject = SendWithoutBlock v4, :dup - v8:BasicObject = SendWithoutBlock v6, :freeze - CheckInterrupts - Return v8 - "#]]); - } - - #[test] - fn test_no_elide_freeze_str_with_args() { - eval(" - def test = ''.freeze(nil) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:StringExact = StringCopy v2 - v5:NilClass = Const Value(nil) - v7:BasicObject = SendWithoutBlock v4, :freeze, v5 - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn test_elide_uminus_with_frozen_str() { - eval(" - def test = -'' - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_elide_uminus_with_refrozen_str() { - eval(" - def test = -''.freeze - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) - PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_no_elide_uminus_with_unfrozen_str() { - eval(" - def test = -''.dup - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:StringExact = StringCopy v2 - v6:BasicObject = SendWithoutBlock v4, :dup - v8:BasicObject = SendWithoutBlock v6, :-@ - CheckInterrupts - Return v8 - "#]]); - } - - #[test] - fn test_objtostring_anytostring_string() { - eval(r##" - def test = "#{('foo')}" - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v3:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - v5:StringExact = StringCopy v3 - v11:StringExact = StringConcat v2, v5 - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_objtostring_anytostring_with_non_string() { - eval(r##" - def test = "#{1}" - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v3:Fixnum[1] = Const Value(1) - v5:BasicObject = ObjToString v3 - v7:String = AnyToString v3, str: v5 - v9:StringExact = StringConcat v2, v7 - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_branchnil_nil() { - eval(" - def test - x = nil - x&.itself - end - "); - - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:NilClass = Const Value(nil) - CheckInterrupts - CheckInterrupts - Return v3 - "#]]); - } - - #[test] - fn test_branchnil_truthy() { - eval(" - def test - x = 1 - x&.itself - end - "); - - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v3:Fixnum[1] = Const Value(1) - CheckInterrupts - PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) - v19:BasicObject = CCall itself@0x1038, v3 - CheckInterrupts - Return v19 - "#]]); - } - - #[test] - fn test_eliminate_load_from_frozen_array_in_bounds() { - eval(r##" - def test = [4,5,6].freeze[1] - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - v5:Fixnum[1] = Const Value(1) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_AREF) - v13:Fixnum[5] = Const Value(5) - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_eliminate_load_from_frozen_array_negative() { - eval(r##" - def test = [4,5,6].freeze[-3] - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - v5:Fixnum[-3] = Const Value(-3) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_AREF) - v13:Fixnum[4] = Const Value(4) - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_eliminate_load_from_frozen_array_negative_out_of_bounds() { - eval(r##" - def test = [4,5,6].freeze[-10] - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - v5:Fixnum[-10] = Const Value(-10) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_AREF) - v13:NilClass = Const Value(nil) - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_eliminate_load_from_frozen_array_out_of_bounds() { - eval(r##" - def test = [4,5,6].freeze[10] - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - v5:Fixnum[10] = Const Value(10) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_AREF) - v13:NilClass = Const Value(nil) - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_dont_optimize_array_aref_if_redefined() { - eval(r##" - class Array - def [](index); end - end - def test = [4,5,6].freeze[10] - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:5: - bb0(v0:BasicObject): - v3:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) - v5:Fixnum[10] = Const Value(10) - v7:BasicObject = SendWithoutBlock v3, :[], v5 - CheckInterrupts - Return v7 - "#]]); - } - - #[test] - fn test_dont_optimize_array_max_if_redefined() { - eval(r##" - class Array - def max = 10 - end - def test = [4,5,6].max - "##); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:5: - bb0(v0:BasicObject): - v2:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - v4:ArrayExact = ArrayDup v2 - PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018) - v11:BasicObject = SendWithoutBlockDirect v4, :max (0x1040) - CheckInterrupts - Return v11 - "#]]); - } - - #[test] - fn test_set_type_from_constant() { - eval(" - MY_SET = Set.new - - def test = MY_SET - - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:4: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, MY_SET) - v9:SetExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_regexp_type() { - eval(" - def test = /a/ - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:RegexpExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) - CheckInterrupts - Return v2 - "#]]); - } - - #[test] - fn test_nil_nil_specialized_to_ccall() { - eval(" - def test = nil.nil? - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:NilClass = Const Value(nil) - PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) - v9:TrueClass = CCall nil?@0x1038, v2 - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_eliminate_nil_nil_specialized_to_ccall() { - eval(" - def test - nil.nil? - 1 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:NilClass = Const Value(nil) - PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) - v5:Fixnum[1] = Const Value(1) - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_non_nil_nil_specialized_to_ccall() { - eval(" - def test = 1.nil? - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) - v9:FalseClass = CCall nil?@0x1038, v2 - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_eliminate_non_nil_nil_specialized_to_ccall() { - eval(" - def test - 1.nil? - 2 - end - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - v2:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) - v5:Fixnum[2] = Const Value(2) - CheckInterrupts - Return v5 - "#]]); - } - - #[test] - fn test_guard_nil_for_nil_opt() { - eval(" - def test(val) = val.nil? - - test(nil) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) - v9:NilClass = GuardType v1, NilClass - v10:TrueClass = CCall nil?@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_guard_false_for_nil_opt() { - eval(" - def test(val) = val.nil? - - test(false) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(FalseClass@0x1000, nil?@0x1008, cme:0x1010) - v9:FalseClass = GuardType v1, FalseClass - v10:FalseClass = CCall nil?@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_guard_true_for_nil_opt() { - eval(" - def test(val) = val.nil? - - test(true) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(TrueClass@0x1000, nil?@0x1008, cme:0x1010) - v9:TrueClass = GuardType v1, TrueClass - v10:FalseClass = CCall nil?@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_guard_symbol_for_nil_opt() { - eval(" - def test(val) = val.nil? - - test(:foo) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(Symbol@0x1000, nil?@0x1008, cme:0x1010) - v9:StaticSymbol = GuardType v1, StaticSymbol - v10:FalseClass = CCall nil?@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_guard_fixnum_for_nil_opt() { - eval(" - def test(val) = val.nil? - - test(1) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) - v9:Fixnum = GuardType v1, Fixnum - v10:FalseClass = CCall nil?@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_guard_float_for_nil_opt() { - eval(" - def test(val) = val.nil? - - test(1.0) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(Float@0x1000, nil?@0x1008, cme:0x1010) - v9:Flonum = GuardType v1, Flonum - v10:FalseClass = CCall nil?@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_guard_string_for_nil_opt() { - eval(" - def test(val) = val.nil? - - test('foo') - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(String@0x1000, nil?@0x1008, cme:0x1010) - v9:StringExact = GuardType v1, StringExact - v10:FalseClass = CCall nil?@0x1038, v9 - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_guard_fixnum_and_fixnum() { - eval(" - def test(x, y) = x & y - - test(1, 2) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, 28) - v10:Fixnum = GuardType v1, Fixnum - v11:Fixnum = GuardType v2, Fixnum - v12:Fixnum = FixnumAnd v10, v11 - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_guard_fixnum_or_fixnum() { - eval(" - def test(x, y) = x | y - - test(1, 2) - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:2: - bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject): - PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, 29) - v10:Fixnum = GuardType v1, Fixnum - v11:Fixnum = GuardType v2, Fixnum - v12:Fixnum = FixnumOr v10, v11 - CheckInterrupts - Return v12 - "#]]); - } - - #[test] - fn test_method_redefinition_patch_point_on_top_level_method() { - eval(" - def foo; end - def test = foo - - test; test - "); - - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:3: - bb0(v0:BasicObject): - PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v8:HeapObject[class_exact*:Object@VALUE(0x1000)] = GuardType v0, HeapObject[class_exact*:Object@VALUE(0x1000)] - v9:BasicObject = SendWithoutBlockDirect v8, :foo (0x1038) - CheckInterrupts - Return v9 - "#]]); - } - - #[test] - fn test_inline_attr_reader_constant() { - eval(" - class C - attr_reader :foo - end - - O = C.new - def test = O.foo - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:7: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, O) - v11:BasicObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) - v13:BasicObject = GetIvar v11, :@foo - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_inline_attr_accessor_constant() { - eval(" - class C - attr_accessor :foo - end - - O = C.new - def test = O.foo - test - test - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:7: - bb0(v0:BasicObject): - PatchPoint SingleRactorMode - PatchPoint StableConstantNames(0x1000, O) - v11:BasicObject[VALUE(0x1008)] = Const Value(VALUE(0x1008)) - PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) - v13:BasicObject = GetIvar v11, :@foo - CheckInterrupts - Return v13 - "#]]); - } - - #[test] - fn test_inline_attr_reader() { - eval(" - class C - attr_reader :foo - end - - def test(o) = o.foo - test C.new - test C.new - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:6: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) - v9:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C] - v10:BasicObject = GetIvar v9, :@foo - CheckInterrupts - Return v10 - "#]]); - } - - #[test] - fn test_inline_attr_accessor() { - eval(" - class C - attr_accessor :foo - end - - def test(o) = o.foo - test C.new - test C.new - "); - assert_optimized_method_hir("test", expect![[r#" - fn test@<compiled>:6: - bb0(v0:BasicObject, v1:BasicObject): - PatchPoint MethodRedefined(C@0x1000, foo@0x1008, cme:0x1010) - v9:HeapObject[class_exact:C] = GuardType v1, HeapObject[class_exact:C] - v10:BasicObject = GetIvar v9, :@foo - CheckInterrupts - Return v10 - "#]]); - } -} diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs new file mode 100644 index 0000000000..9a2ff9f03b --- /dev/null +++ b/zjit/src/hir/opt_tests.rs @@ -0,0 +1,17408 @@ +#[cfg(test)] +mod hir_opt_tests { + use crate::hir::*; + + use crate::{hir_strings, options::*}; + use insta::assert_snapshot; + use crate::hir::tests::hir_build_tests::assert_contains_opcode; + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[test] + fn test_fold_iftrue_away() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v25:Fixnum[3] = Const Value(3) + Return v25 + "); + } + + #[test] + fn test_fold_iftrue_into_jump() { + eval(" + def test + cond = false + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:FalseClass = Const Value(false) + CheckInterrupts + v35:Fixnum[4] = Const Value(4) + Return v35 + "); + } + + #[test] + fn test_fold_fixnum_add() { + eval(" + def test + 1 + 2 + 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, +@0x1008, cme:0x1010) + v33:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_add_zero() { + eval(" + def test(n) + 0 + n + 0 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v32:Fixnum = GuardType v10, Fixnum + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_fold_fixnum_sub() { + eval(" + def test + 5 - 3 - 1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v33:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_sub_large_negative_result() { + eval(" + def test + 0 - 1073741825 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[1073741825] = Const Value(1073741825) + PatchPoint MethodRedefined(Integer@0x1000, -@0x1008, cme:0x1010) + v24:Fixnum[-1073741825] = Const Value(-1073741825) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_sub_zero() { + eval(" + def test(n) + n - 0 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1008, -@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_fixnum_mult() { + eval(" + def test + 6 * 7 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[6] = Const Value(6) + v12:Fixnum[7] = Const Value(7) + PatchPoint MethodRedefined(Integer@0x1000, *@0x1008, cme:0x1010) + v24:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mult_zero() { + eval(" + def test(n) + 0 * n + n * 0 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1008, *@0x1010, cme:0x1018) + v36:Fixnum = GuardType v10, Fixnum + v46:Fixnum[0] = Const Value(0) + v47:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1040, cme:0x1048) + CheckInterrupts + Return v47 + "); + } + + #[test] + fn test_fold_fixnum_mult_one() { + eval(" + def test(n) + 1 * n + n * 1 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, *@0x1010, cme:0x1018) + v36:Fixnum = GuardType v10, Fixnum + PatchPoint MethodRedefined(Integer@0x1008, +@0x1040, cme:0x1048) + v45:Fixnum = FixnumAdd v36, v36 + CheckInterrupts + Return v45 + "); + } + + #[test] + fn test_fold_fixnum_div() { + eval(" + def test + 7 / 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_fold_fixnum_div_zero() { + eval(" + def test + 7 / 0 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v23:Integer = FixnumDiv v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_div_negative() { + eval(" + def test + 7 / -3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[-3] = Const Value(-3) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v24:Fixnum[-3] = Const Value(-3) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_fold_fixnum_div_negative_one_overflow() { + eval(&format!(" + def test + {RUBY_FIXNUM_MIN} / -1 + end + ")); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-4611686018427387904] = Const Value(-4611686018427387904) + v12:Fixnum[-1] = Const Value(-1) + PatchPoint MethodRedefined(Integer@0x1000, /@0x1008, cme:0x1010) + v23:Integer = FixnumDiv v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_div_one() { + eval(" + def test(n) + n / 1 + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, /@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_fixnum_mod_zero_by_zero() { + eval(" + def test + 0 % 0 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v23:Fixnum = FixnumMod v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_mod_non_zero_by_zero() { + eval(" + def test + 11 % 0 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[11] = Const Value(11) + v12:Fixnum[0] = Const Value(0) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v23:Fixnum = FixnumMod v10, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_mod_zero_by_non_zero() { + eval(" + def test + 0 % 11 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[11] = Const Value(11) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod() { + eval(" + def test + 11 % 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[11] = Const Value(11) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod_negative_numerator() { + eval(" + def test + -7 % 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-7] = Const Value(-7) + v12:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod_negative_denominator() { + eval(" + def test + 7 % -3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[7] = Const Value(7) + v12:Fixnum[-3] = Const Value(-3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[-2] = Const Value(-2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_mod_negative() { + eval(" + def test + -7 % -3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-7] = Const Value(-7) + v12:Fixnum[-3] = Const Value(-3) + PatchPoint MethodRedefined(Integer@0x1000, %@0x1008, cme:0x1010) + v24:Fixnum[-1] = Const Value(-1) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_xor() { + eval(" + def test + 2 ^ 5 + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v23:Fixnum[7] = Const Value(7) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_xor_same_negative_number() { + eval(" + def test + 123 ^ -123 + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[123] = Const Value(123) + v12:Fixnum[-123] = Const Value(-123) + PatchPoint MethodRedefined(Integer@0x1000, ^@0x1008, cme:0x1010) + v23:Fixnum[-2] = Const Value(-2) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_fold_fixnum_and() { + eval(" + def test + 4 & -7 + end + "); + + assert_snapshot!(inspect("test"), @"0"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[-7] = Const Value(-7) + PatchPoint MethodRedefined(Integer@0x1000, &@0x1008, cme:0x1010) + v25:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_and_with_negative_self() { + eval(" + def test + -4 & 7 + end + "); + + assert_snapshot!(inspect("test"), @"4"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-4] = Const Value(-4) + v12:Fixnum[7] = Const Value(7) + PatchPoint MethodRedefined(Integer@0x1000, &@0x1008, cme:0x1010) + v25:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_or() { + eval(" + def test + 4 | 1 + end + "); + + assert_snapshot!(inspect("test"), @"5"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v25:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_or_with_negative_self() { + eval(" + def test + -4 | 1 + end + "); + + assert_snapshot!(inspect("test"), @"-3"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[-4] = Const Value(-4) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v25:Fixnum[-3] = Const Value(-3) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_or_with_negative_other() { + eval(" + def test + 4 | -1 + end + "); + + assert_snapshot!(inspect("test"), @"-1"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[-1] = Const Value(-1) + PatchPoint MethodRedefined(Integer@0x1000, |@0x1008, cme:0x1010) + v25:Fixnum[-1] = Const Value(-1) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_fold_fixnum_less() { + eval(" + def test + if 1 < 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_less_equal() { + eval(" + def test + if 1 <= 2 && 2 <= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, <=@0x1008, cme:0x1010) + v58:TrueClass = Const Value(true) + CheckInterrupts + v37:Fixnum[3] = Const Value(3) + Return v37 + "); + } + + #[test] + fn test_fold_fixnum_greater() { + eval(" + def test + if 2 > 1 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_greater_equal() { + eval(" + def test + if 2 >= 1 && 2 >= 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, >=@0x1008, cme:0x1010) + v58:TrueClass = Const Value(true) + CheckInterrupts + v37:Fixnum[3] = Const Value(3) + Return v37 + "); + } + + #[test] + fn test_fold_fixnum_eq_false() { + eval(" + def test + if 1 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v42:FalseClass = Const Value(false) + CheckInterrupts + v33:Fixnum[4] = Const Value(4) + Return v33 + "); + } + + #[test] + fn test_fold_fixnum_eq_true() { + eval(" + def test + if 2 == 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, ==@0x1008, cme:0x1010) + v42:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_neq_true() { + eval(" + def test + if 1 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v43:TrueClass = Const Value(true) + CheckInterrupts + v24:Fixnum[3] = Const Value(3) + Return v24 + "); + } + + #[test] + fn test_fold_fixnum_neq_false() { + eval(" + def test + if 2 != 2 + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[2] = Const Value(2) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, !=@0x1008, cme:0x1010) + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v43:FalseClass = Const Value(false) + CheckInterrupts + v33:Fixnum[4] = Const Value(4) + Return v33 + "); + } + + #[test] + fn test_fold_unbox_fixnum() { + eval(" + def test(arr) = arr[0] + test([1,2,3]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact recompile + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v27 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v27, v30 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_guard_greater_eq() { + eval(" + def test(arr) = arr[0] + test([1,2,3]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact recompile + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v27 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v27, v30 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_fold_guard_greater_eq_side_exit() { + eval(r##" + def test = [4,5,6].freeze[-10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-10] = Const Value(-10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v32:CInt64[-10] = Const CInt64(-10) + v33:CInt64[3] = Const CInt64(3) + v28:CInt64 = AdjustBounds v32, v33 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64 = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v11, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn neq_with_side_effect_not_elided () { + let result = eval(" + class CustomEq + attr_reader :count + + def ==(o) + @count = @count.to_i + 1 + self.equal?(o) + end + end + + def test(object) + # intentionally unused, but also can't assign to underscore + object != object + nil + end + + custom = CustomEq.new + test(custom) + test(custom) + + custom.count + "); + assert_eq!(VALUE::fixnum_from_usize(2), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:13: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :object@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :object@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(CustomEq@0x1008) + PatchPoint MethodRedefined(CustomEq@0x1008, !=@0x1010, cme:0x1018) + v30:ObjectSubclass[class_exact:CustomEq] = GuardType v10, ObjectSubclass[class_exact:CustomEq] recompile + v31:BoolExact = CCallWithFrame v30, :BasicObject#!=@0x1040, v30 + v21:NilClass = Const Value(nil) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_replace_guard_if_known_fixnum() { + eval(" + def test(a) + a + 1 + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:Fixnum = FixnumAdd v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_param_forms_get_bb_param() { + eval(" + def rest(*array) = array + def kw(k:) = k + def kw_rest(**k) = k + def post(*rest, post) = post + def block(&b) = nil + "); + assert_snapshot!(hir_strings!("rest", "kw", "kw_rest", "block", "post"), @" + fn rest@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :array@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :array@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + Return v10 + + fn kw@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :k@0x1000 + v4:BasicObject = LoadField v2, :<empty>@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :k@1 + v9:CPtr = GetEP 0 + v10:BasicObject = LoadField v9, :<empty>@0x1002 + Jump bb3(v7, v8, v10) + bb3(v12:BasicObject, v13:BasicObject, v14:BasicObject): + CheckInterrupts + Return v13 + + fn kw_rest@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :k@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :k@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + Return v10 + + fn block@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :b@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :b@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + CheckInterrupts + Return v14 + + fn post@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :rest@0x1000 + v4:BasicObject = LoadField v2, :post@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :rest@1 + v9:BasicObject = LoadArg :post@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_optimize_top_level_call_into_send_direct() { + eval(" + def foo = [] + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_iseq() { + eval(" + def foo = 1 + alias bar foo + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_send_without_block_to_aliased_cfunc() { + eval(" + alias bar itself + alias baz bar + def test = baz + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, baz@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_inline_nonparam_local_return() { + // Methods that return non-parameter local variables should NOT be inlined, + // because the local variable index will be out of bounds for args. + // The method must have a parameter so param_size > 0, and return a local + // that's not a parameter so local_idx >= param_size. + // Use dead code (if false) to create a local without initialization instructions, + // resulting in just getlocal + leave which enters the inlining code path. + eval(" + def foo(a) + if false + x = nil + end + x + end + def test = foo(1) + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_to_aliased_cfunc() { + eval(" + class C < Array + alias fun_new_map map + end + def test(o) = o.fun_new_map {|e| e } + test C.new; test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, fun_new_map@0x1010, cme:0x1018) + v27:ArraySubclass[class_exact:C] = GuardType v10, ArraySubclass[class_exact:C] recompile + v28:BasicObject = SendDirect v27, 0x1040, :fun_new_map (0x1050) + PatchPoint NoEPEscape(test) + v18:CPtr = LoadSP + v19:BasicObject = LoadField v18, :o@0x1000 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_send_to_aliased_cfunc_from_module() { + eval(" + class C + include Enumerable + def each; yield 1; end + alias bar map + end + def test(o) = o.bar { |x| x } + test C.new; test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, bar@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v29:BasicObject = CCallWithFrame v28, :Enumerable#bar@0x1040, block=0x1048 + PatchPoint NoEPEscape(test) + v18:CPtr = LoadSP + v19:BasicObject = LoadField v18, :o@0x1000 + CheckInterrupts + Return v29 + "); + } + + // Regression test: when specialized_instruction is disabled, the compiler + // doesn't convert `send` to `opt_send_without_block`, so a no-block call + // reaches ZJIT as `YARVINSN_send` with a null blockiseq. This becomes + // `Send { blockiseq: Some(null_ptr) }` which must be normalized to None in + // reduce_send_to_ccall, otherwise CCallWithFrame gens wrong block handler. + #[test] + fn test_send_to_cfunc_without_specialized_instruction() { + eval_with_options(" + def test(a) = a.length + test([1,2,3]); test([1,2,3]) + ", "{ specialized_instruction: false }"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, length@0x1010, cme:0x1018) + v24:ArrayExact = GuardType v10, ArrayExact recompile + v25:CInt64 = ArrayLength v24 + v26:Fixnum = BoxFixnum v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_nonexistent_top_level_call() { + eval(" + def foo + end + def test + foo + end + test; test + undef :foo + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, :foo # SendFallbackReason: Send: unsupported method type Null + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_private_top_level_call() { + eval(" + def foo = [] + private :foo + def test + foo + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_top_level_call_with_overloaded_cme() { + eval(" + def test + Integer(3) + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v21:BasicObject = SendDirect v20, 0x1038, :Integer (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_top_level_call_with_args_into_send_direct() { + eval(" + def foo(a, b) = [] + def test + foo 1, 2 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_top_level_sends_into_send_direct() { + eval(" + def foo = [] + def bar = [] + def test + foo + bar + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v23:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v24:BasicObject = SendDirect v23, 0x1038, :foo (0x1048) + PatchPoint MethodRedefined(Object@0x1000, bar@0x1050, cme:0x1058) + v27:BasicObject = SendDirect v23, 0x1038, :bar (0x1048) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_send_direct_no_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_send_direct_one_optional_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_send_direct_all_optionals_passed() { + eval(" + def foo(a=1, b=2) = a + b + def test = foo 3, 4 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[4] = Const Value(4) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_call_with_correct_and_too_many_args_for_method() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4) = [a, b, c, d] + def test = [target(), target(10, 20, 30), begin; target(10, 20, 30, 40, 50) rescue ArgumentError; end] + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v44:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v45:BasicObject = SendDirect v44, 0x1038, :target (0x1048) + v14:Fixnum[10] = Const Value(10) + v16:Fixnum[20] = Const Value(20) + v18:Fixnum[30] = Const Value(30) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:BasicObject = SendDirect v44, 0x1038, :target (0x1048), v14, v16, v18 + v24:Fixnum[10] = Const Value(10) + v26:Fixnum[20] = Const Value(20) + v28:Fixnum[30] = Const Value(30) + v30:Fixnum[40] = Const Value(40) + v32:Fixnum[50] = Const Value(50) + v34:BasicObject = Send v44, :target, v24, v26, v28, v30, v32 # SendFallbackReason: Argument count does not match parameter count + v37:ArrayExact = NewArray v45, v48, v34 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_optimize_variadic_ccall() { + eval(" + def test + puts 'Hello' + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + PatchPoint MethodRedefined(Object@0x1008, puts@0x1010, cme:0x1018) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v23:BasicObject = CCallVariadic v22, :Kernel#puts@0x1040, v12 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_optimize_fixnum_add_if_redefined() { + eval(" + class Integer + def +(other) + 100 + end + end + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum recompile + v29:Fixnum[100] = Const Value(100) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_both_profiled() { + eval(" + def test(a, b) = a + b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum recompile + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumAdd v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_left_profiled() { + eval(" + def test(a) = a + 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:Fixnum = FixnumAdd v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_add_right_profiled() { + eval(" + def test(a) = 1 + a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v27:Fixnum = GuardType v10, Fixnum + v28:Fixnum = FixnumAdd v14, v27 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn integer_aref_with_fixnum_emits_fixnum_aref() { + eval(" + def test(a, b) = a[b] + test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, []@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum recompile + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumAref v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn elide_fixnum_aref() { + eval(" + def test + 1[2] + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_too_many_args() { + eval(" + def test = 1[2, 3] + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v14:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Integer@0x1000, []@0x1008, cme:0x1010) + v24:BasicObject = CCallVariadic v10, :Integer#[]@0x1038, v12, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn do_not_optimize_integer_aref_with_non_fixnum() { + eval(r#" + def test = 1["x"] + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint MethodRedefined(Integer@0x1008, []@0x1010, cme:0x1018) + v24:BasicObject = CCallVariadic v10, :Integer#[]@0x1040, v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_both_profiled() { + eval(" + def test(a, b) = a < b + test(1,2); test(3,4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum recompile + v29:Fixnum = GuardType v13, Fixnum + v30:BoolExact = FixnumLt v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_left_profiled() { + eval(" + def test(a) = a < 1 + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:BoolExact = FixnumLt v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_send_into_fixnum_lt_right_profiled() { + eval(" + def test(a) = 1 < a + test(1); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v27:Fixnum = GuardType v10, Fixnum + v28:BoolExact = FixnumLt v14, v27 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_literals() { + eval(" + def test() + a = 2 + (1..a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v26:RangeExact = NewRangeFixnum v17 NewRangeInclusive v13 + CheckInterrupts + Return v26 + "); + } + + + #[test] + fn test_optimize_new_range_fixnum_exclusive_literals() { + eval(" + def test() + a = 2 + (1...a) + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[2] = Const Value(2) + v17:Fixnum[1] = Const Value(1) + v26:RangeExact = NewRangeFixnum v17 NewRangeExclusive v13 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_high_guarded() { + eval(" + def test(a) + (1..a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v14 NewRangeInclusive v23 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_high_guarded() { + eval(" + def test(a) + (1...a) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v14 NewRangeExclusive v23 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_inclusive_low_guarded() { + eval(" + def test(a) + (a..10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v23 NewRangeInclusive v15 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_new_range_fixnum_exclusive_low_guarded() { + eval(" + def test(a) + (a...10) + end + test(2); test(3) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v23:Fixnum = GuardType v10, Fixnum + v24:RangeExact = NewRangeFixnum v23 NewRangeExclusive v15 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_eliminate_comment() { + let mut function = Function::new(std::ptr::null()); + let block = function.entry_block; + + let comment = function.push_comment(block, "diagnostic".to_string()); + let dead_const = function.push_insn(block, Insn::Const { val: Const::CBool(false) }); + let return_val = function.push_insn(block, Insn::Const { val: Const::CBool(true) }); + function.push_insn(block, Insn::Return { val: return_val }); + function.seal_entries(); + + function.eliminate_dead_code(); + + let insns = &function.blocks[block.0].insns; + assert!(insns.contains(&comment)); + assert!(!insns.contains(&dead_const)); + } + + #[test] + fn test_eliminate_new_array() { + eval(" + def test() + c = [] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_opt_aref_array() { + eval(" + arr = [1,2,3] + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact recompile + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v27 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v27, v30 + CheckInterrupts + Return v34 + "); + assert_snapshot!(inspect("test [1,2,3]"), @"1"); + } + + #[test] + fn test_opt_aref_hash() { + eval(" + arr = {0 => 4} + def test(arr) = arr[0] + test(arr) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v27:HashExact = GuardType v10, HashExact recompile + v28:BasicObject = HashAref v27, v15 + CheckInterrupts + Return v28 + "); + assert_snapshot!(inspect("test({0 => 4})"), @"4"); + } + + #[test] + fn test_eliminate_new_range() { + eval(" + def test() + c = (1..2) + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:RangeExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_do_not_eliminate_new_range_non_fixnum() { + eval(" + def test() + _ = (-'a'..'b') + 0 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v14:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v16:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v17:StringExact = StringCopy v16 + v19:RangeExact = NewRange v14 NewRangeInclusive v17 + PatchPoint NoEPEscape(test) + v25:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_eliminate_new_array_with_elements() { + eval(" + def test(a) + c = [a] + 5 + end + test(1); test(2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v18:ArrayExact = NewArray v12 + v22:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_eliminate_new_hash() { + eval(" + def test() + c = {} + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_no_eliminate_new_hash_with_elements() { + eval(" + def test(aval, bval) + c = {a: aval, b: bval} + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :aval@0x1000 + v4:BasicObject = LoadField v2, :bval@0x1001 + v5:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :aval@1 + v10:BasicObject = LoadArg :bval@2 + v11:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:NilClass): + v20:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + v23:StaticSymbol[:b] = Const Value(VALUE(0x1010)) + v26:HashExact = NewHash v20: v14, v23: v15 + PatchPoint NoEPEscape(test) + v32:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_eliminate_array_dup() { + eval(" + def test + c = [1, 2] + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_hash_dup() { + eval(" + def test + c = {a: 1, b: 2} + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_putself() { + eval(" + def test() + c = self + 5 + end + test; test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_eliminate_string_copy() { + eval(r#" + def test() + c = "abc" + 5 + end + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact = StringCopy v13 + v18:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_eliminate_fixnum_add() { + eval(" + def test(a, b) + a + b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_sub() { + eval(" + def test(a, b) + a - b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, -@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_mul() { + eval(" + def test(a, b) + a * b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, *@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_div() { + eval(" + def test(a, b) + a / b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, /@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v34:Integer = FixnumDiv v32, v33 + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_eliminate_fixnum_mod() { + eval(" + def test(a, b) + a % b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, %@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v34:Fixnum = FixnumMod v32, v33 + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_lt() { + eval(" + def test(a, b) + a < b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_le() { + eval(" + def test(a, b) + a <= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <=@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_gt() { + eval(" + def test(a, b) + a > b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, >@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_ge() { + eval(" + def test(a, b) + a >= b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, >=@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_eq() { + eval(" + def test(a, b) + a == b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ==@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + v33:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_eliminate_fixnum_neq() { + eval(" + def test(a, b) + a != b + 5 + end + test(1, 2); test(3, 4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, !=@0x1010, cme:0x1018) + v32:Fixnum = GuardType v12, Fixnum recompile + PatchPoint BOPRedefined(INTEGER_REDEFINED_OP_FLAG, BOP_EQ) + v34:Fixnum = GuardType v13, Fixnum + v24:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_do_not_eliminate_get_constant_path() { + eval(" + def test() + C + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v14:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_do_not_eliminate_getconstant() { + eval(" + def test(klass) + klass::ARGV + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :klass@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :klass@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:FalseClass = Const Value(false) + v17:BasicObject = GetConstant v10, :ARGV, v15 + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn kernel_itself_const() { + eval(" + def test(x) = x.itself + test(0) # profile + test(1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, itself@0x1010, cme:0x1018) + v23:Fixnum = GuardType v10, Fixnum recompile + CheckInterrupts + Return v23 + "); + } + + #[test] + fn kernel_itself_known_type() { + eval(" + def test = [].itself + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn eliminate_kernel_itself() { + eval(" + def test + x = [].itself + 1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, itself@0x1008, cme:0x1010) + PatchPoint NoEPEscape(test) + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn eliminate_module_name() { + eval(" + module M; end + def test + x = M.name + 1 + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v29:ModuleExact[M@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, name@0x1018, cme:0x1020) + v34:StringExact|NilClass = CCall v29, :Module#name@0x1048 + PatchPoint NoEPEscape(test) + v21:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn eliminate_array_length() { + eval(" + def test + [].length + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, length@0x1008, cme:0x1010) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn normal_class_type_inference() { + eval(" + class C; end + def test = C + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v18:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn core_classes_type_inference() { + eval(" + def test = [String, Class, Module, BasicObject] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v26:ClassSubclass[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Class) + v29:ClassSubclass[Class@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint StableConstantNames(0x1020, Module) + v32:ClassSubclass[Module@0x1028] = Const Value(VALUE(0x1028)) + PatchPoint StableConstantNames(0x1030, BasicObject) + v35:ClassSubclass[BasicObject@0x1038] = Const Value(VALUE(0x1038)) + v18:ArrayExact = NewArray v26, v29, v32, v35 + CheckInterrupts + Return v18 + "); + } + + #[test] + fn module_instances_are_module_exact() { + eval(" + def test = [Enumerable, Kernel] + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Enumerable) + v22:ModuleExact[Enumerable@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Kernel) + v25:ModuleSubclass[Kernel@0x1018] = Const Value(VALUE(0x1018)) + v14:ArrayExact = NewArray v22, v25 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn module_subclasses_are_not_module_exact() { + eval(" + class ModuleSubclass < Module; end + MY_MODULE = ModuleSubclass.new + def test = MY_MODULE + test # Warm the constant cache + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_MODULE) + v18:ModuleSubclass[MY_MODULE@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn eliminate_array_size() { + eval(" + def test + [].size + 5 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, size@0x1008, cme:0x1010) + v17:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn kernel_itself_argc_mismatch() { + eval(" + def test = 1.itself(0) + test rescue 0 + test rescue 0 + "); + // Not specialized + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[0] = Const Value(0) + v14:BasicObject = Send v10, :itself, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_kernel_block_given_p() { + eval(" + def test = block_given? + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:CPtr = GetEP 0 + v21:BoolExact = IsBlockGiven v20 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_inline_kernel_block_given_p_in_block() { + eval(" + TEST = proc { block_given? } + TEST.call + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:FalseClass = Const Value(false) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_elide_kernel_block_given_p() { + eval(" + def test + block_given? + 5 + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, block_given?@0x1008, cme:0x1010) + v23:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v15:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn const_send_direct_integer() { + eval(" + def test(x) = 1.zero? + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, zero?@0x1010, cme:0x1018) + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v14 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn class_known_send_direct_array() { + eval(" + def test(x) + a = [1,2,3] + a.first + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v18:ArrayExact = ArrayDup v17 + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, first@0x1018, cme:0x1020) + v32:BasicObject = InvokeBuiltin leaf <inline_expr>, v18 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn send_direct_to_module() { + eval(" + module M; end + def test = M.class + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, M) + v20:ModuleExact[M@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Module@0x1010) + PatchPoint MethodRedefined(Module@0x1010, class@0x1018, cme:0x1020) + v26:ClassSubclass[Module@0x1010] = Const Value(VALUE(0x1010)) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_to_instance_method() { + eval(" + class C + def foo = [] + end + + def test(c) = c.foo + c = C.new + test c + test c + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :c@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :c@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v24:BasicObject = SendDirect v23, 0x1040, :foo (0x1050) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_send_direct_iseq_with_block() { + let result = eval(" + def foo(a, b, &block) = block.call(a, b) + def test = foo(1, 2) { |a, b| a + b } + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(3), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn reload_local_across_send() { + eval(" + def foo(&block) = 1 + def test + a = 1 + foo {|| } + a + end + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v34:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v36:Fixnum[1] = Const Value(1) + PatchPoint NoEPEscape(test) + v21:CPtr = LoadSP + v22:BasicObject = LoadField v21, :a@0x1038 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn reload_local_across_send_after_ep_escape() { + eval(" + def foo(&block) = 1 + def test + a = 1 + lambda { a } + foo {|| } + a + end + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + SetLocal :a, l0, EP@3, v13 + PatchPoint MethodRedefined(Object@0x1000, lambda@0x1008, cme:0x1010) + v45:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v46:BasicObject = CCallWithFrame v45, :Kernel#lambda@0x1038, block=0x1040 + v20:CPtr = GetEP 0 + v21:BasicObject = LoadField v20, :a@0x1048 + PatchPoint MethodRedefined(Object@0x1000, foo@0x1049, cme:0x1050) + v32:CPtr = GetEP 0 + v33:BasicObject = LoadField v32, :a@0x1048 + CheckInterrupts + Return v33 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_rest() { + enable_zjit_stats(); + eval(" + def foo(*args) = 1 + def test = foo 1 + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v14:Fixnum[1] = Const Value(1) + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_param_rest + v17:BasicObject = Send v7, :foo, v14 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v17 + "); + } + + #[test] + fn specialize_call_to_post_param_iseq() { + eval(" + def foo(opt=80, post) = post + def test = foo(10) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[10] = Const Value(10) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn specialize_call_to_iseq_with_optional_between_required_params() { + let result = eval(" + def foo(lead, opt=80, post) = lead + opt + post + def test = foo(10, 20) + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(110), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[10] = Const Value(10) + v13:Fixnum[20] = Const Value(20) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_to_iseq_with_multiple_required_kw() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_to_iseq_with_required_kw_reorder() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v26:BasicObject = SendDirect v25, 0x1038, :foo (0x1048), v13, v15, v11 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_to_iseq_with_positional_and_required_kw_reorder() { + eval(" + def foo(x, a:, b:) = [x, a, b] + def test = foo(0, b: 2, a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v26:BasicObject = SendDirect v25, 0x1038, :foo (0x1048), v11, v15, v13 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_with_positional_and_optional_kw() { + eval(" + def foo(x, a: 1) = [x, a] + def test = foo(0, a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[0] = Const Value(0) + v13:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v23:BasicObject = SendDirect v22, 0x1038, :foo (0x1048), v11, v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn specialize_call_with_pos_optional_and_req_kw() { + eval(" + def foo(r, x = 2, a:, b:) = [x, a] + def test = [foo(1, a: 3, b: 4), foo(1, 2, b: 4, a: 3)] # with and without the optional, change kw order + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + v15:Fixnum[4] = Const Value(4) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v38:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v11, v13, v15 + v20:Fixnum[1] = Const Value(1) + v22:Fixnum[2] = Const Value(2) + v24:Fixnum[4] = Const Value(4) + v26:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v42:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v20, v22, v26, v24 + v30:ArrayExact = NewArray v38, v42 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn specialize_call_with_pos_optional_and_kw_optional() { + eval(" + def foo(r, x = 2, a:, b: 4) = [r, x, a, b] + def test = [foo(1, a: 3), foo(1, 2, b: 40, a: 30)] # with and without the optionals + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[3] = Const Value(3) + v34:Fixnum[4] = Const Value(4) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v38:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v11, v13, v34 + v18:Fixnum[1] = Const Value(1) + v20:Fixnum[2] = Const Value(2) + v22:Fixnum[40] = Const Value(40) + v24:Fixnum[30] = Const Value(30) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v42:BasicObject = SendDirect v37, 0x1038, :foo (0x1048), v18, v20, v24, v22 + v28:ArrayExact = NewArray v38, v42 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_call_with_pos_optional_and_maybe_too_many_args() { + eval(" + def target(a = 1, b = 2, c = 3, d = 4, e = 5, f:) = [a, b, c, d, e, f] + def test = [target(f: 6), target(10, 20, 30, f: 6), target(10, 20, 30, 40, 50, f: 60)] + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[6] = Const Value(6) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v48:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v49:BasicObject = SendDirect v48, 0x1038, :target (0x1048), v11 + v16:Fixnum[10] = Const Value(10) + v18:Fixnum[20] = Const Value(20) + v20:Fixnum[30] = Const Value(30) + v22:Fixnum[6] = Const Value(6) + PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) + v52:BasicObject = SendDirect v48, 0x1038, :target (0x1048), v16, v18, v20, v22 + v27:Fixnum[10] = Const Value(10) + v29:Fixnum[20] = Const Value(20) + v31:Fixnum[30] = Const Value(30) + v33:Fixnum[40] = Const Value(40) + v35:Fixnum[50] = Const Value(50) + v37:Fixnum[60] = Const Value(60) + v39:BasicObject = Send v48, :target, v27, v29, v31, v33, v35, v37 # SendFallbackReason: Too many arguments for LIR + v41:ArrayExact = NewArray v49, v52, v39 + CheckInterrupts + Return v41 + "); + } + + #[test] + fn test_send_call_to_iseq_with_optional_kw() { + eval(" + def foo(a: 1) = a + def test = foo(a: 2) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v11 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_kwrest() { + enable_zjit_stats(); + eval(" + def foo(**args) = 1 + def test = foo(a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v14:Fixnum[1] = Const Value(1) + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_param_kwrest + v17:BasicObject = Send v7, :foo, v14 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_send_hash_to_kwarg_only_method() { + eval(r#" + def callee(a:) = a + def test = callee({a: 1}) + begin; test; rescue ArgumentError; end + begin; test; rescue ArgumentError; end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:HashExact = HashDup v11 + v14:BasicObject = Send v6, :callee, v12 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_send_hash_to_optional_kwarg_only_method() { + eval(r#" + def callee(a: nil) = a + def test = callee({a: 1}) + begin; test; rescue ArgumentError; end + begin; test; rescue ArgumentError; end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:HashExact = HashDup v11 + v14:BasicObject = Send v6, :callee, v12 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v14 + "); + } + + #[test] + fn specialize_call_to_iseq_with_optional_param_kw_using_default() { + eval(" + def foo(int: 1) = int + 1 + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v17:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v21:BasicObject = SendDirect v20, 0x1038, :foo (0x1048), v17 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_call_kwsplat() { + enable_zjit_stats(); + eval(" + def foo(a:) = a + def test = foo(**{a: 1}) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v14:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v15:HashExact = HashDup v14 + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_caller_kw_splat + v18:BasicObject = Send v7, :foo, v15 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_param_kwrest() { + enable_zjit_stats(); + eval(" + def foo(**kwargs) = kwargs.keys + def test = foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + IncrCounterPtr + Jump bb3(v4) + bb3(v7:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter complex_arg_pass_param_kwrest + v14:BasicObject = Send v7, :foo # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v14 + "); + } + + #[test] + fn dont_optimize_ccall_with_kwarg() { + eval(" + def test = sprintf('%s', a: 1) + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact = StringCopy v11 + v14:Fixnum[1] = Const Value(1) + v16:BasicObject = Send v6, :sprintf, v12, v14 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v16 + "); + } + + #[test] + fn dont_optimize_ccall_with_block_and_kwarg() { + eval(" + def test(s) + a = [] + s.each_line(chomp: true) { |l| a << l } + a + end + test %(a\nb\nc) + test %() + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:ArrayExact = NewArray + v22:TrueClass = Const Value(true) + v24:BasicObject = Send v12, 0x1008, :each_line, v22 # SendFallbackReason: Complex argument passing + PatchPoint NoEPEscape(test) + v27:CPtr = LoadSP + v28:BasicObject = LoadField v27, :s@0x1000 + v29:BasicObject = LoadField v27, :a@0x1030 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_empty_ic() { + eval(" + def test = Kernel + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn dont_replace_get_constant_path_with_invalidated_ic() { + eval(" + def test = Kernel + test + Kernel = 5 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn replace_get_constant_path_with_const() { + eval(" + def test = Kernel + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Kernel) + v18:ModuleSubclass[Kernel@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn replace_nested_get_constant_path_with_const() { + eval(" + module Foo + module Bar + class C + end + end + end + def test = Foo::Bar::C + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo::Bar::C) + v18:ClassSubclass[Foo::Bar::C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_opt_new_no_initialize() { + eval(" + class C; end + def test = C.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v43:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v46:ObjectSubclass[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v51:NilClass = Const Value(nil) + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_initialize() { + eval(" + class C + def initialize x + @x = x + end + end + def test = C.new 1 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v46:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(C@0x1008, new@0x1009, cme:0x1010) + v49:ObjectSubclass[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) + v53:BasicObject = SendDirect v49, 0x1068, :initialize (0x1078), v15 + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_opt_new_object() { + eval(" + def test = Object.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Object) + v43:ClassSubclass[Object@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Object@0x1008, new@0x1009, cme:0x1010) + v46:ObjectExact = ObjectAllocClass Object:VALUE(0x1008) + PatchPoint NoSingletonClass(Object@0x1008) + PatchPoint MethodRedefined(Object@0x1008, initialize@0x1038, cme:0x1040) + v51:NilClass = Const Value(nil) + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_basic_object() { + eval(" + def test = BasicObject.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, BasicObject) + v43:ClassSubclass[BasicObject@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(BasicObject@0x1008, new@0x1009, cme:0x1010) + v46:BasicObjectExact = ObjectAllocClass BasicObject:VALUE(0x1008) + PatchPoint NoSingletonClass(BasicObject@0x1008) + PatchPoint MethodRedefined(BasicObject@0x1008, initialize@0x1038, cme:0x1040) + v51:NilClass = Const Value(nil) + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_opt_new_hash() { + eval(" + def test = Hash.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Hash) + v43:ClassSubclass[Hash@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Hash@0x1008, new@0x1009, cme:0x1010) + v46:HashExact = ObjectAllocClass Hash:VALUE(0x1008) + v47:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, initialize@0x1038, cme:0x1040) + v52:BasicObject = SendDirect v46, 0x1068, :initialize (0x1078), v47 + CheckInterrupts + Return v46 + "); + assert_snapshot!(inspect("test"), @"{}"); + } + + #[test] + fn test_opt_new_array() { + eval(" + def test = Array.new 1 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Array) + v46:ClassSubclass[Array@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Array@0x1008, new@0x1009, cme:0x1010) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v53:BasicObject = CCallVariadic v46, :Array.new@0x1040, v15 + CheckInterrupts + Return v53 + "); + } + + #[test] + fn test_opt_new_set() { + eval(" + def test = Set.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Set) + v43:ClassSubclass[Set@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(Set@0x1008, new@0x1009, cme:0x1010) + v17:HeapBasicObject = ObjectAlloc v43 + PatchPoint NoSingletonClass(Set@0x1008) + PatchPoint MethodRedefined(Set@0x1008, initialize@0x1038, cme:0x1040) + v49:SetExact = GuardType v17, SetExact recompile + v50:BasicObject = CCallVariadic v49, :Set#initialize@0x1068 + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_opt_new_string() { + eval(" + def test = String.new + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v43:ClassSubclass[String@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + PatchPoint MethodRedefined(String@0x1008, new@0x1009, cme:0x1010) + PatchPoint MethodRedefined(Class@0x1038, new@0x1009, cme:0x1010) + v54:BasicObject = CCallVariadic v43, :String.new@0x1040 + CheckInterrupts + Return v54 + "); + } + + #[test] + fn test_opt_new_regexp() { + eval(" + def test = Regexp.new '' + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Regexp) + v47:ClassSubclass[Regexp@0x1008] = Const Value(VALUE(0x1008)) + v12:NilClass = Const Value(nil) + v15:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v16:StringExact = StringCopy v15 + PatchPoint MethodRedefined(Regexp@0x1008, new@0x1018, cme:0x1020) + v50:RegexpExact = ObjectAllocClass Regexp:VALUE(0x1008) + PatchPoint NoSingletonClass(Regexp@0x1008) + PatchPoint MethodRedefined(Regexp@0x1008, initialize@0x1048, cme:0x1050) + v55:BasicObject = CCallVariadic v50, :Regexp#initialize@0x1078, v16 + CheckInterrupts + Return v50 + "); + } + + #[test] + fn test_inline_class_allocate() { + eval(" + class C; end + def test = C.allocate + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v20:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, allocate@0x1018, cme:0x1020) + v24:ObjectSubclass[class_exact:C] = ObjectAllocClass C:VALUE(0x1008) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_inline_class_allocate_with_args() { + eval(" + class C; end + def test = C.allocate(1) + test rescue 0 + test rescue 0 + "); + // Not specialized + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, C) + v22:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + v12:Fixnum[1] = Const Value(1) + v14:BasicObject = Send v22, :allocate, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_dont_inline_class_allocate_with_singleton_class() { + eval(" + class C; end + SC = C.singleton_class + def test = SC.allocate + test rescue 0 + "); + // Not specialized: singleton classes are not leaf allocators + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, SC) + v20:ClassSubclass[Class@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, allocate@0x1018, cme:0x1020) + v24:BasicObject = CCallWithFrame v20, :Class.allocate@0x1048 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, length@0x1010, cme:0x1018) + v31:CInt64 = ArrayLength v19 + v32:Fixnum = BoxFixnum v31 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, size@0x1010, cme:0x1018) + v31:CInt64 = ArrayLength v19 + v32:Fixnum = BoxFixnum v31 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_getblockparamproxy() { + eval(" + def test(&block) = tap(&block) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:CPtr = GetEP 0 + v18:CUInt64 = LoadField v17, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v19:CBool = IsBlockParamModified v18 + CondBranch v19, bb4(), bb5() + bb4(): + v21:BasicObject = LoadField v17, :block@0x1002 + Jump bb6(v21, v21) + bb5(): + v23:CInt64 = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) + v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v25, v10) + bb6(v15:BasicObject, v16:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_getblockparamproxy_modified() { + eval(" + def test(&block) + b = block + tap(&block) + end + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :block@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22) + bb5(): + v24:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb6(v24) + bb6(v17:BasicObject): + v32:CPtr = GetEP 0 + v33:CUInt64 = LoadField v32, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v34:CBool = IsBlockParamModified v33 + CondBranch v34, bb7(), bb8() + bb7(): + v36:BasicObject = LoadField v32, :block@0x1002 + Jump bb9(v36, v36) + bb8(): + v38:CInt64 = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) + v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v40, v17) + bb9(v30:BasicObject, v31:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_getblockparamproxy_modified_nested_block() { + eval(" + def test(&block) + proc do + b = block + tap(&block) + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v14:CPtr = GetEP 1 + v15:CUInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CBool = IsBlockParamModified v15 + CondBranch v16, bb4(), bb5() + bb4(): + v18:BasicObject = LoadField v14, :block@0x1001 + Jump bb6(v18) + bb5(): + v20:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v20) + bb6(v13:BasicObject): + v27:CPtr = GetEP 1 + v28:CUInt64 = LoadField v27, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v29:CBool = IsBlockParamModified v28 + CondBranch v29, bb7(), bb8() + bb7(): + v31:BasicObject = LoadField v27, :block@0x1001 + Jump bb9(v31) + bb8(): + v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) + v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v35) + bb9(v26:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb9() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb9(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb8(), bb10() + bb8(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb6(v16:BasicObject, v17:BasicObject): + v38:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v38 + bb10(): + SideExit BlockParamProxyProfileNotCovered + "); + } + + #[test] + fn test_getblockparam() { + eval(" + def test(&block) = block + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CPtr = GetEP 0 + v16:CUInt64 = LoadField v15, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v17:CBool = IsBlockParamModified v16 + CondBranch v17, bb4(), bb5() + bb4(): + v19:BasicObject = LoadField v15, :block@0x1002 + Jump bb6(v19) + bb5(): + v21:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb6(v21) + bb6(v14:BasicObject): + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_getblockparam_nested_block() { + eval(" + def test(&block) + proc do + block + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:CPtr = GetEP 1 + v12:CUInt64 = LoadField v11, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v13:CBool = IsBlockParamModified v12 + CondBranch v13, bb4(), bb5() + bb4(): + v15:BasicObject = LoadField v11, :block@0x1001 + Jump bb6(v15) + bb5(): + v17:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v17) + bb6(v10:BasicObject): + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setblockparam() { + eval(" + def test(&block) + block = nil + end + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + SetLocal :block, l0, EP@3, v14 + v18:CPtr = GetEP 0 + v19:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CInt64[512] = Const CInt64(512) + v21:CInt64 = IntOr v19, v20 + StoreField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001, v21 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_setblockparam_nested_block() { + eval(" + def test(&block) + proc do + block = nil + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + SetLocal :block, l1, EP@3, v10 + v14:CPtr = GetEP 1 + v15:CInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CInt64[512] = Const CInt64(512) + v17:CInt64 = IntOr v15, v16 + StoreField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000, v17 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_true() { + eval(" + @foo = 4 + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) recompile + v19:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_specialize_monomorphic_definedivar_false() { + eval(" + def test = defined?(@foo) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v16:HeapBasicObject = GuardType v6, HeapBasicObject + v17:CShape = LoadField v16, :shape_id@0x1000 + v18:CShape[0x1001] = GuardBitEquals v17, CShape(0x1001) recompile + v19:NilClass = Const Value(nil) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_specialize_proc_call() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.call(1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, call@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] recompile + v26:BasicObject = InvokeProc v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_proc_aref() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p[2] + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[2] = Const Value(2) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, []@0x1010, cme:0x1018) + v26:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] recompile + v27:BasicObject = InvokeProc v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_specialize_proc_yield() { + eval(" + p = proc { |x| x + 1 } + def test(p) + p.yield(3) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, yield@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] recompile + v26:BasicObject = InvokeProc v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_proc_eqq() { + eval(" + p = proc { |x| x > 0 } + def test(p) + p === 1 + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, ===@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:Proc] = GuardType v10, ObjectSubclass[class_exact:Proc] recompile + v26:BasicObject = InvokeProc v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_specialize_proc_call_splat() { + eval(" + p = proc { } + def test(p) + empty = [] + p.call(*empty) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :p@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:ArrayExact = NewArray + v23:ArrayExact = ToArray v17 + v25:BasicObject = Send v12, :call, v23 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_specialize_proc_call_kwarg() { + eval(" + p = proc { |a:| a } + def test(p) + p.call(a: 1) + end + test p + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :p@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :p@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v17:BasicObject = Send v10, :call, v15 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_dont_specialize_definedivar_with_immediate() { + eval(" + module M + def test = defined?(@a) + end + + class Integer + include M + end + + 1.test + 2.test + TEST = M.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_definedivar_with_t_struct() { + // Range is T_STRUCT (not T_OBJECT): falls back to DefinedIvar. + eval(" + class C < Range + def test = defined?(@a) + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_optimize_definedivar_polymorphic() { + set_call_threshold(3); + eval(" + class C + def test = defined?(@a) + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HeapBasicObject = GuardType v6, HeapBasicObject + v11:CUInt64 = LoadField v10, :RBASIC_FLAGS@0x1000 + v13:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v14:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v15 = RefineType v14, CUInt64 + v16:CInt64 = IntAnd v11, v13 + v17:CBool = IsBitEqual v16, v15 + CondBranch v17, bb5(), bb6() + bb5(): + v19:NilClass = Const Value(nil) + Jump bb4(v19) + bb6(): + v21:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v22:CPtr[CPtr(0x1002)] = Const CPtr(0x1002) + v23 = RefineType v22, CUInt64 + v24:CInt64 = IntAnd v11, v21 + v25:CBool = IsBitEqual v24, v23 + CondBranch v25, bb7(), bb8() + bb7(): + v27:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + Jump bb4(v27) + bb8(): + v29:StringExact|NilClass = DefinedIvar v10, :@a + Jump bb4(v29) + bb4(v12:StringExact|NilClass): + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_optimize_definedivar_polymorphic_with_immediate() { + set_call_threshold(3); + eval(r#" + module M + def test = defined?(@a) + end + + class C + include M + end + + class Integer + include M + end + + obj = C.new + obj.instance_variable_set(:@a, 1) + + obj.test + 1.test + TEST = M.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HeapBasicObject = GuardType v6, HeapBasicObject + v11:CUInt64 = LoadField v10, :RBASIC_FLAGS@0x1000 + v13:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v14:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v15 = RefineType v14, CUInt64 + v16:CInt64 = IntAnd v11, v13 + v17:CBool = IsBitEqual v16, v15 + CondBranch v17, bb5(), bb6() + bb5(): + v19:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + Jump bb4(v19) + bb6(): + v21:StringExact|NilClass = DefinedIvar v10, :@a + Jump bb4(v21) + bb4(v12:StringExact|NilClass): + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_optimize_definedivar_polymorphic_with_t_struct() { + set_call_threshold(3); + eval(r#" + module M + def test = defined?(@a) + end + + class C + include M + end + + class D < Range + include M + end + + obj = C.new + obj.instance_variable_set(:@a, 1) + + range = D.new 0, 1 + range.instance_variable_set(:@a, 1) + + obj.test + range.test + TEST = M.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HeapBasicObject = GuardType v6, HeapBasicObject + v11:CUInt64 = LoadField v10, :RBASIC_FLAGS@0x1000 + v13:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v14:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v15 = RefineType v14, CUInt64 + v16:CInt64 = IntAnd v11, v13 + v17:CBool = IsBitEqual v16, v15 + CondBranch v17, bb5(), bb6() + bb5(): + v19:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + Jump bb4(v19) + bb6(): + v21:StringExact|NilClass = DefinedIvar v10, :@a + Jump bb4(v21) + bb4(v12:StringExact|NilClass): + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_optimize_definedivar_polymorphic_with_complex_shape() { + set_call_threshold(3); + eval(r#" + module M + def test = defined?(@a) + end + + class C + include M + end + + class D + include M + end + + obj = C.new + obj.instance_variable_set(:@a, 1) + + complex = D.new + (0..1000).each do |i| + complex.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + complex.remove_instance_variable(:"@v#{i}") + end + + obj.test + complex.test + TEST = M.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HeapBasicObject = GuardType v6, HeapBasicObject + v11:CUInt64 = LoadField v10, :RBASIC_FLAGS@0x1000 + v13:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v14:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v15 = RefineType v14, CUInt64 + v16:CInt64 = IntAnd v11, v13 + v17:CBool = IsBitEqual v16, v15 + CondBranch v17, bb5(), bb6() + bb5(): + v19:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + Jump bb4(v19) + bb6(): + v21:StringExact|NilClass = DefinedIvar v10, :@a + Jump bb4(v21) + bb4(v12:StringExact|NilClass): + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_definedivar() { + eval(r#" + class C + def test = defined?(@a) + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@a + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_already_in_shape() { + eval(" + @foo = 4 + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v21:HeapBasicObject = GuardType v6, HeapBasicObject + v22:CShape = LoadField v21, :shape_id@0x1000 + v23:CShape[0x1001] = GuardBitEquals v22, CShape(0x1001) recompile + StoreField v21, :@foo@0x1002, v10 + WriteBarrier v21, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_monomorphic_setivar_with_shape_transition() { + eval(" + def test = @foo = 5 + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + v21:HeapBasicObject = GuardType v6, HeapBasicObject + v22:CShape = LoadField v21, :shape_id@0x1000 + v23:CShape[0x1001] = GuardBitEquals v22, CShape(0x1001) recompile + StoreField v21, :@foo@0x1002, v10 + WriteBarrier v21, v10 + v26:CShape[0x1003] = Const CShape(0x1003) + StoreField v21, :shape_id@0x1000, v26 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_specialize_multiple_monomorphic_setivar_with_shape_transition() { + eval(r#" + klass = Class.new do + def test + @foo = 1 + @bar = 2 + end + end + + # Grow class max_iv_count so fresh instances can keep both writes + # on the embedded fast path. + warm = klass.new + warm.instance_variable_set(:@warm1, 1) + warm.instance_variable_set(:@warm2, 2) + + obj = klass.new + obj.test + TEST = klass.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v28:HeapBasicObject = GuardType v6, HeapBasicObject + v29:CShape = LoadField v28, :shape_id@0x1000 + v30:CShape[0x1001] = GuardBitEquals v29, CShape(0x1001) recompile + StoreField v28, :@foo@0x1002, v10 + WriteBarrier v28, v10 + v33:CShape[0x1003] = Const CShape(0x1003) + StoreField v28, :shape_id@0x1000, v33 + v17:Fixnum[2] = Const Value(2) + PatchPoint SingleRactorMode + StoreField v28, :@bar@0x1004, v17 + WriteBarrier v28, v17 + v40:CShape[0x1005] = Const CShape(0x1005) + StoreField v28, :shape_id@0x1000, v40 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_dont_specialize_setivar_with_t_data() { + eval(" + class C < Range + def test = @a = 5 + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_polymorphic_setivar() { + set_call_threshold(3); + eval(" + class C + def test = @a = 5 + end + obj = C.new + obj.instance_variable_set(:@a, 1) + obj.test + obj = C.new + obj.instance_variable_set(:@b, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_complex_shape_setivar() { + eval(r#" + class C + def test = @a = 5 + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + obj.test + TEST = C.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@a, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_dont_specialize_setivar_when_next_shape_is_complex() { + eval(r#" + class AboutToBeTooComplex + def test = @abc = 5 + end + SHAPE_MAX_VARIATIONS = 8 # see shape.h + SHAPE_MAX_VARIATIONS.times do + AboutToBeTooComplex.new.instance_variable_set(:"@a#{_1}", 1) + end + AboutToBeTooComplex.new.test + TEST = AboutToBeTooComplex.instance_method(:test) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@abc, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_hash() { + eval(" + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_dont_optimize_hash_freeze_if_redefined() { + eval(" + class Hash + def freeze; end + end + def test = {}.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_hash() { + eval(" + def test = {}.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_hash() { + eval(" + def test = {}.dup.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact = NewHash + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, dup@0x1008, cme:0x1010) + v23:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = Send v23, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_hash_with_args() { + eval(" + def test = {}.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact = NewHash + v12:NilClass = Const Value(nil) + v14:BasicObject = Send v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_ary() { + eval(" + def test = [].freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_ary() { + eval(" + def test = [].freeze.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_ary() { + eval(" + def test = [].dup.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, dup@0x1008, cme:0x1010) + v23:BasicObject = CCallWithFrame v10, :Kernel#dup@0x1038 + v14:BasicObject = Send v23, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_no_elide_freeze_ary_with_args() { + eval(" + def test = [].freeze(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + v12:NilClass = Const Value(nil) + v14:BasicObject = Send v10, :freeze, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_elide_freeze_with_frozen_str() { + eval(" + def test = ''.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_freeze_with_refrozen_str() { + eval(" + def test = ''.freeze.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_freeze_with_unfrozen_str() { + eval(" + def test = ''.dup.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v24:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = Send v24, :freeze # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_no_elide_freeze_str_with_args() { + eval(" + def test = ''.freeze(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:NilClass = Const Value(nil) + v15:BasicObject = Send v11, :freeze, v13 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_elide_uminus_with_frozen_str() { + eval(" + def test = -'' + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_uminus_with_refrozen_str() { + eval(" + def test = -''.freeze + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_elide_uminus_with_unfrozen_str() { + eval(" + def test = -''.dup + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, dup@0x1010, cme:0x1018) + v24:BasicObject = CCallWithFrame v11, :String#dup@0x1040 + v15:BasicObject = Send v24, :-@ # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_objtostring_anytostring_string() { + eval(r##" + def test = "#{('foo')}" + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v14:StringExact = StringCopy v13 + v21:StringExact = StringConcat v10, v14 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_objtostring_anytostring_with_non_string() { + eval(r##" + def test = "#{1}" + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled() { + eval(" + def test(a) + \"#{a}\" + end + test('foo'); test('foo') + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + v29:String = GuardType v10, String + v22:StringExact = StringConcat v14, v29 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_objtostring_anytostring_recv_profiled_string_subclass() { + eval(" + class MyString < String; end + + def test(a) + \"#{a}\" + end + foo = MyString.new('foo') + test(MyString.new(foo)); test(MyString.new(foo)) + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(MyString@0x1010) + v29:String = GuardType v10, String + v22:StringExact = StringConcat v14, v29 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_objtostring_profiled_nonstring_falls_back_to_send() { + eval(" + def test(a) + \"#{a}\" + end + test([1,2,3]); test([1,2,3]) # No fast path for array + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v28:ArrayExact = GuardType v10, ArrayExact + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, to_s@0x1018, cme:0x1020) + v34:BasicObject = CCallWithFrame v28, :Array#to_s@0x1048 + v20:String = AnyToString v28, str: v34 + v22:StringExact = StringConcat v14, v20 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_branchnil_nil() { + eval(" + def test + x = nil + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + CheckInterrupts + v21:NilClass = Const Value(nil) + Return v21 + "); + } + + #[test] + fn test_branchnil_truthy() { + eval(" + def test + x = 1 + x&.itself + end + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + PatchPoint MethodRedefined(Integer@0x1000, itself@0x1008, cme:0x1010) + Return v13 + "); + } + + #[test] + fn test_dont_eliminate_load_from_non_frozen_array() { + eval(r##" + S = [4,5,6] + def test = S[0] + test + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v23:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, []@0x1018, cme:0x1020) + v35:CInt64[0] = Const CInt64(0) + v29:CInt64 = ArrayLength v23 + v30:CInt64[0] = GuardLess v35, v29 + v34:BasicObject = ArrayAref v23, v30 + CheckInterrupts + Return v34 + "); + // TODO(max): Check the result of `S[0] = 5; test` using `inspect` to make sure that we + // actually do the load at run-time. + } + + #[test] + fn test_eliminate_load_from_frozen_array_in_bounds() { + eval(r##" + def test = [4,5,6].freeze[1] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v34:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative() { + eval(r##" + def test = [4,5,6].freeze[-3] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-3] = Const Value(-3) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v32:CInt64[-3] = Const CInt64(-3) + v33:CInt64[3] = Const CInt64(3) + v28:CInt64 = AdjustBounds v32, v33 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64 = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v11, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_negative_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[-10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[-10] = Const Value(-10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v32:CInt64[-10] = Const CInt64(-10) + v33:CInt64[3] = Const CInt64(3) + v28:CInt64 = AdjustBounds v32, v33 + v29:CInt64[0] = Const CInt64(0) + v30:CInt64 = GuardGreaterEq v28, v29 + v31:BasicObject = ArrayAref v11, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_eliminate_load_from_frozen_array_out_of_bounds() { + eval(r##" + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + SideExit GuardLess + "); + } + + #[test] + fn test_dont_optimize_array_aref_if_redefined() { + eval(r##" + class Array + def [](index) = [] + end + def test = [4,5,6].freeze[10] + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v24:BasicObject = SendDirect v11, 0x1040, :[] (0x1050), v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_dont_optimize_array_aset_if_redefined() { + eval(r##" + class Array + def []=(*args); :redefined; end + end + + def test(arr) + arr[1] = 10 + end + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[1] = Const Value(1) + v19:Fixnum[10] = Const Value(10) + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_dont_optimize_array_max_if_redefined() { + eval(r##" + class Array + def max = [] + end + def test = [4,5,6].max + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018) + v21:BasicObject = SendDirect v11, 0x1040, :max (0x1050) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_set_type_from_constant() { + eval(" + MY_SET = Set.new + + def test = MY_SET + + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MY_SET) + v18:SetExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_regexp_type() { + eval(" + def test = /a/ + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:RegexpExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_bmethod_send_direct() { + eval(" + define_method(:zero) { :b } + define_method(:one) { |arg| arg } + + def test = one(zero) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint MethodRedefined(Object@0x1000, zero@0x1008, cme:0x1010) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v28:StaticSymbol[:b] = Const Value(VALUE(0x1038)) + PatchPoint MethodRedefined(Object@0x1000, one@0x1040, cme:0x1048) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_symbol_block_bmethod() { + eval(" + define_method(:identity, &:itself) + def test = identity(100) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[100] = Const Value(100) + v13:BasicObject = Send v6, :identity, v11 # SendFallbackReason: Bmethod: Proc object is not defined by an ISEQ + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_call_bmethod_with_block() { + eval(" + define_method(:bmethod) { :b } + def test = (bmethod {}) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :bmethod # SendFallbackReason: Send: unsupported method type Bmethod + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_call_shareable_bmethod() { + eval(" + class Foo + class << self + define_method(:identity, &(Ractor.make_shareable ->(val){val})) + end + end + def test = Foo.identity(100) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Foo) + v22:ClassSubclass[Foo@0x1008] = Const Value(VALUE(0x1008)) + v12:Fixnum[100] = Const Value(100) + PatchPoint MethodRedefined(Class@0x1010, identity@0x1018, cme:0x1020) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_nil_nil_specialized_to_ccall() { + eval(" + def test = nil.nil? + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v21:TrueClass = Const Value(true) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_eliminate_nil_nil_specialized_to_ccall() { + eval(" + def test + nil.nil? + 1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + PatchPoint MethodRedefined(NilClass@0x1000, nil?@0x1008, cme:0x1010) + v17:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_non_nil_nil_specialized_to_ccall() { + eval(" + def test = 1.nil? + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v21:FalseClass = Const Value(false) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_eliminate_non_nil_nil_specialized_to_ccall() { + eval(" + def test + 1.nil? + 2 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, nil?@0x1008, cme:0x1010) + v17:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_guard_nil_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1008, nil?@0x1010, cme:0x1018) + v24:NilClass = GuardType v10, NilClass recompile + v25:TrueClass = Const Value(true) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_false_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(false) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1008, nil?@0x1010, cme:0x1018) + v24:FalseClass = GuardType v10, FalseClass recompile + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_true_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(true) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1008, nil?@0x1010, cme:0x1018) + v24:TrueClass = GuardType v10, TrueClass recompile + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_symbol_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(:foo) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1008, nil?@0x1010, cme:0x1018) + v24:StaticSymbol = GuardType v10, StaticSymbol recompile + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_fixnum_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, nil?@0x1010, cme:0x1018) + v24:Fixnum = GuardType v10, Fixnum recompile + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_float_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test(1.0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, nil?@0x1010, cme:0x1018) + v24:Flonum = GuardType v10, Flonum recompile + v25:FalseClass = Const Value(false) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_guard_string_for_nil_opt() { + eval(" + def test(val) = val.nil? + + test('foo') + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, nil?@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact recompile + v26:FalseClass = Const Value(false) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_basicobject_not_truthy() { + eval(" + def test(a) = !a + + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, !@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact recompile + v26:FalseClass = Const Value(false) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_basicobject_not_false() { + eval(" + def test(a) = !a + + test(false) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(FalseClass@0x1008, !@0x1010, cme:0x1018) + v24:FalseClass = GuardType v10, FalseClass recompile + v25:TrueClass = Const Value(true) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_basicobject_not_nil() { + eval(" + def test(a) = !a + + test(nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(NilClass@0x1008, !@0x1010, cme:0x1018) + v24:NilClass = GuardType v10, NilClass recompile + v25:TrueClass = Const Value(true) + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_specialize_basicobject_not_falsy() { + eval(" + def test(a) = !(if a then false else nil end) + + # TODO(max): Make this not GuardType NilClass and instead just reason + # statically + test(false) + test(true) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb6(), bb4(v9, v17) + bb6(): + v19:Truthy = RefineType v10, Truthy + v21:FalseClass = Const Value(false) + CheckInterrupts + Jump bb5(v9, v19, v21) + bb4(v25:BasicObject, v26:Falsy): + v29:NilClass = Const Value(nil) + Jump bb5(v25, v26, v29) + bb5(v31:BasicObject, v32:BasicObject, v33:Falsy): + v37:CBool = HasType v33, FalseClass + CondBranch v37, bb8(v31, v32, v33), bb9() + bb8(v38:BasicObject, v39:BasicObject, v40:Falsy): + PatchPoint MethodRedefined(FalseClass@0x1008, !@0x1010, cme:0x1018) + v68:TrueClass = Const Value(true) + Jump bb7(v38, v39, v68) + bb9(): + v46:CBool = HasType v33, NilClass + CondBranch v46, bb10(v31, v32, v33), bb11() + bb10(v47:BasicObject, v48:BasicObject, v49:Falsy): + PatchPoint MethodRedefined(NilClass@0x1040, !@0x1010, cme:0x1018) + v71:TrueClass = Const Value(true) + Jump bb7(v47, v48, v71) + bb11(): + v55:BasicObject = Send v33, :! # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb7(v31, v32, v55) + bb7(v57:BasicObject, v58:BasicObject, v59:BasicObject): + CheckInterrupts + Return v59 + "); + } + + #[test] + fn test_specialize_array_empty_p() { + eval(" + def test(a) = a.empty? + + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, empty?@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact recompile + v26:CInt64 = ArrayLength v25 + v27:CInt64[0] = Const CInt64(0) + v28:CBool = IsBitEqual v26, v27 + v29:BoolExact = BoxBool v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_specialize_hash_empty_p_to_ccall() { + eval(" + def test(a) = a.empty? + + test({}) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, empty?@0x1010, cme:0x1018) + v25:HashExact = GuardType v10, HashExact recompile + v26:BoolExact = CCall v25, :Hash#empty?@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_basic_object_eq_to_ccall() { + eval(" + class C; end + def test(a, b) = a == b + + test(C.new, C.new) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, ==@0x1010, cme:0x1018) + v29:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] recompile + v30:CBool = IsBitEqual v29, v13 + v31:BoolExact = BoxBool v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_guard_fixnum_and_fixnum() { + eval(" + def test(x, y) = x & y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, &@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum recompile + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumAnd v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_guard_fixnum_or_fixnum() { + eval(" + def test(x, y) = x | y + + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, |@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum recompile + v29:Fixnum = GuardType v13, Fixnum + v30:Fixnum = FixnumOr v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_method_redefinition_patch_point_on_top_level_method() { + eval(" + def foo; end + def test = foo + + test; test + "); + + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_getivar_embedded() { + eval(" + class C + attr_reader :foo + def initialize + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:BasicObject = LoadField v23, :@foo@0x1042 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_getivar_complex() { + eval(r#" + class C + attr_reader :foo + def initialize + 1000.times do |i| + instance_variable_set("@v#{i}", i) + end + @foo = 42 + end + end + + O = C.new + def test(o) = o.foo + test O + test O + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:13: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v24:BasicObject = GetIvar v23, :@foo + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_getivar_shape_guard_recompile() { + // Call with one shape to compile, then call with a different shape to + // trigger shape guard exits and recompilation. On the recompiled version, + // GetIvar stays as a C call because iseq_to_hir handles polymorphic + // branching at parse time for getinstancevariable. + eval(" + class C + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + def foo = @foo + end + + c = C.new + c.foo # profile + c.foo # compile (version 1 with shape guard) + d = C.new(true) # same class, different shape + 100.times { d.foo } # trigger shape guard exits -> recompile + 100.times { c.foo } # run recompiled version (version 2) + "); + // After recompilation, iseq_to_hir generates polymorphic branches at + // parse time using the exit-profiled shapes: two optimized LoadField + // fast paths plus a GetIvar C call fallback. + assert_snapshot!(hir_string_proc("C.new.method(:foo)"), @" + fn foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:HeapBasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:HeapBasicObject): + PatchPoint SingleRactorMode + v12:CUInt64 = LoadField v6, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:BasicObject = LoadField v6, :@foo@0x1002 + Jump bb4(v20) + bb6(): + v22:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v23:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v24 = RefineType v23, CUInt64 + v25:CInt64 = IntAnd v12, v22 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:BasicObject = LoadField v6, :@foo@0x1004 + Jump bb4(v28) + bb8(): + v30:BasicObject = GetIvar v6, :@foo + Jump bb4(v30) + bb4(v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + // The following tests pin down the soundness boundary of the `self: + // HeapBasicObject` inference (see `iseq_self_is_heap_object`). A `def` method + // gets `self: HeapBasicObject` only when its owning class can never produce an + // immediate receiver. For each class below, `self` must stay `BasicObject`: + // the six immediate classes have no default allocator, and Object/BasicObject/ + // Numeric use the default allocator but are ancestors of immediates (caught by + // the Integer kind_of check). Each test reopens the class, compiles the method + // (call threshold is 30), then checks the resulting `self` type. + + #[test] + fn test_self_not_heap_object_owner_integer() { + eval(" + class Integer + def probe = @foo + end + 100.times { 5.probe } + "); + assert_snapshot!(hir_string_proc("5.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_self_not_heap_object_owner_symbol() { + eval(" + class Symbol + def probe = @foo + end + 100.times { :sym.probe } + "); + assert_snapshot!(hir_string_proc(":sym.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_self_not_heap_object_owner_float() { + eval(" + class Float + def probe = @foo + end + 100.times { 1.5.probe } + "); + assert_snapshot!(hir_string_proc("1.5.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_self_not_heap_object_owner_nil_class() { + eval(" + class NilClass + def probe = @foo + end + 100.times { nil.probe } + "); + assert_snapshot!(hir_string_proc("nil.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_self_not_heap_object_owner_true_class() { + eval(" + class TrueClass + def probe = @foo + end + 100.times { true.probe } + "); + assert_snapshot!(hir_string_proc("true.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_self_not_heap_object_owner_false_class() { + eval(" + class FalseClass + def probe = @foo + end + 100.times { false.probe } + "); + assert_snapshot!(hir_string_proc("false.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_self_not_heap_object_owner_object() { + // Object uses the default allocator, but Integer (and every other immediate) + // descends from it, so a method on Object can run with an immediate self. + eval(" + class Object + def probe = @foo + end + o = Object.new + 100.times { o.probe } + "); + assert_snapshot!(hir_string_proc("Object.new.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:HeapBasicObject = GuardType v6, HeapBasicObject + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_self_not_heap_object_owner_basic_object() { + // Same as Object: BasicObject has the default allocator but is the root of + // the immediate classes' ancestry. + eval(" + class BasicObject + def probe = @foo + end + o = Object.new + 100.times { o.probe } + "); + assert_snapshot!(hir_string_proc("Object.new.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:HeapBasicObject = GuardType v6, HeapBasicObject + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_self_not_heap_object_owner_numeric() { + // Numeric has the default allocator but Integer/Float descend from it, so a + // method on Numeric can run with an immediate self. + eval(" + class Numeric + def probe = @foo + end + 100.times { 5.probe } + "); + assert_snapshot!(hir_string_proc("5.method(:probe)"), @" + fn probe@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_definedivar_shape_guard_recompile() { + // Call with one shape to compile, then call with a different shape to + // trigger shape guard exits and recompilation. On the recompiled version, + // DefinedIvar uses polymorphic fast paths plus a C call fallback. + eval(" + class C + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + def has_foo = defined?(@foo) + end + + c = C.new + c.has_foo # profile + c.has_foo # compile (version 1 with shape guard) + d = C.new(true) # same class, different shape + 100.times { d.has_foo } # trigger shape guard exits -> recompile + 100.times { c.has_foo } # run recompiled version (version 2) + "); + assert_snapshot!(hir_string_proc("C.new.method(:has_foo)"), @" + fn has_foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:HeapBasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:HeapBasicObject): + v11:CUInt64 = LoadField v6, :RBASIC_FLAGS@0x1000 + v13:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v14:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v15 = RefineType v14, CUInt64 + v16:CInt64 = IntAnd v11, v13 + v17:CBool = IsBitEqual v16, v15 + CondBranch v17, bb5(), bb6() + bb5(): + v19:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + Jump bb4(v19) + bb6(): + v21:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v22:CPtr[CPtr(0x1010)] = Const CPtr(0x1010) + v23 = RefineType v22, CUInt64 + v24:CInt64 = IntAnd v11, v21 + v25:CBool = IsBitEqual v24, v23 + CondBranch v25, bb7(), bb8() + bb7(): + v27:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + Jump bb4(v27) + bb8(): + v29:StringExact|NilClass = DefinedIvar v6, :@foo + Jump bb4(v29) + bb4(v12:StringExact|NilClass): + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_setivar_shape_guard_recompile() { + // Call with one shape to compile, then call with a different shape to + // trigger shape guard exits and recompilation. On the recompiled version, + // SetIvar stays as a C call fallback to avoid more shape guard exits. + eval(" + class C + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + def foo = @foo = 5 + end + + c = C.new + c.foo # profile + c.foo # compile (version 1 with shape guard) + d = C.new(true) # same class, different shape + 100.times { d.foo } # trigger shape guard exits -> recompile + 100.times { c.foo } # run recompiled version (version 2) + "); + assert_snapshot!(hir_string_proc("C.new.method(:foo)"), @" + fn foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:HeapBasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:HeapBasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setivar_shape_guard_attr_writer_no_recompile() { + // attr_writer SetIvar has no inline cache and may target a receiver + // operand other than CFP self, so don't recompile here yet. + eval(" + class C + attr_writer :foo + def initialize(extra = false) + @bar = 0 if extra # changes the shape + @foo = 42 + end + end + + class D + def write(obj) + obj.foo = 5 + end + end + + c = C.new + d = D.new + d.write(c) # profile + d.write(c) # compile (version 1 with shape guard) + e = C.new(true) # same class, different shape + 100.times { d.write(e) } # shape guard exits, but no recompile + "); + assert_snapshot!(hir_string_proc("D.new.method(:write)"), @" + fn write@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:HeapBasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :obj@1 + Jump bb3(v6, v7) + bb3(v9:HeapBasicObject, v10:BasicObject): + v17:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v31:CShape = LoadField v28, :shape_id@0x1040 + v32:CShape[0x1041] = GuardBitEquals v31, CShape(0x1041) + StoreField v28, :@foo@0x1042, v17 + WriteBarrier v28, v17 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_optimize_getivar_on_module_embedded() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:Module = GuardType v6, Module + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + PatchPoint RootBoxOnly + v21:RubyValue = LoadField v17, :fields_obj@0x1002 + v22:BasicObject = LoadField v21, :@foo@0x1003 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_getivar_on_module_complex() { + eval(r#" + module M + @foo = 42 + for i in 0...1000 + instance_variable_set("@v#{i}", i) + end + def self.test = @foo + end + M.test + "#); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_side_exit_assertion() { + eval(" + def side_exit = ::RubyVM::ZJIT.induce_side_exit! + side_exit + "); + std::panic::catch_unwind(|| assert_compiles("side_exit")).expect_err("Should panic because the program should side exit"); + } + + #[test] + fn test_optimize_getivar_on_class_embedded() { + eval(" + class C + @foo = 42 + def self.test = @foo + end + C.test + "); + assert_snapshot!(assert_compiles("C.test"), @"42"); + assert_snapshot!(hir_string_proc("C.method(:test)"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:Class = GuardType v6, Class + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + PatchPoint RootBoxOnly + v21:RubyValue = LoadField v17, :fields_obj@0x1002 + v22:BasicObject = LoadField v21, :@foo@0x1003 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_getivar_on_class_complex() { + eval(r#" + class C + @foo = 42 + for i in 0...1000 + instance_variable_set("@v#{i}", i) + end + def self.test = @foo + end + C.test + "#); + assert_snapshot!(hir_string_proc("C.method(:test)"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_getivar_on_t_struct() { + // Range is T_STRUCT (not T_DATA): falls back to CCall + eval(" + class C < Range + def test = @a + end + obj = C.new 0, 1 + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:HeapBasicObject = GuardType v6, HeapBasicObject + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + v20:CAttrIndex[0] = Const CAttrIndex(0) + v21:BasicObject = CCall v17, :rb_ivar_get_at_no_ractor_check@0x1008, v20 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_getivar_on_t_data() { + // T_DATA uses fields_obj for instance variables. + eval(" + class C < Thread + def test = @a + end + obj = C.new { } + obj.join + obj.instance_variable_set(:@a, 1) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v17:TData = GuardType v6, TData + v18:CShape = LoadField v17, :shape_id@0x1000 + v19:CShape[0x1001] = GuardBitEquals v18, CShape(0x1001) recompile + v20:RubyValue = LoadField v17, :fields_obj@0x1002 + v21:BasicObject = LoadField v20, :@a@0x1002 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_optimize_getivar_on_t_data_complex_fields() { + // T_DATA with enough ivars to force heap field storage + eval(" + class C < Thread + def test = @var1000 + end + obj = C.new { } + obj.join + 1000.times { |i| obj.instance_variable_set(:\"@var#{i}\", 1) } + obj.instance_variable_set(:@var1000, 42) + obj.test + TEST = C.instance_method(:test) + "); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@var1000 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_getivar_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + def self.test = @foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit UnhandledYARVInsn(getinstancevariable) + "); + } + + #[test] + fn test_optimize_attr_reader_on_module_multi_ractor() { + eval(" + module M + @foo = 42 + class << self + attr_reader :foo + end + def self.test = foo + end + Ractor.new {}.value + M.test + "); + assert_snapshot!(hir_string_proc("M.method(:test)"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, :foo # SendFallbackReason: Single-ractor mode required + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_optimize_getivar_polymorphic() { + set_call_threshold(3); + eval(r#" + class C + def foo_then_many + @foo = 1 + 10.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 2 + end + + def many_then_foo + 10.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 3 + @foo = 4 + end + + def foo = @foo + 1 + end + + O1 = C.new + O1.foo_then_many + O2 = C.new + O2.many_then_foo + O1.foo + O2.foo + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:foo)"), @" + fn foo@<compiled>:15: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:BasicObject = LoadField v11, :@foo@0x1002 + Jump bb4(v20) + bb6(): + v22:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v23:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v24 = RefineType v23, CUInt64 + v25:CInt64 = IntAnd v12, v22 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:CPtr = LoadField v11, :as_heap@0x1004 + v29:BasicObject = LoadField v28, :@foo@0x1000 + Jump bb4(v29) + bb8(): + v31:BasicObject = GetIvar v11, :@foo + Jump bb4(v31) + bb4(v13:BasicObject): + v34:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v45:Fixnum = GuardType v13, Fixnum recompile + v46:Fixnum = FixnumAdd v45, v34 + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_optimize_getivar_skewed_polymorphic() { + // Use threshold=6 so we get 5 profile samples. + // 4 calls with shape A, 1 with shape B = 80% skew (>= 75% threshold). + set_call_threshold(6); + eval(r#" + class C + def foo_then_many + @foo = 1 + 100.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 2 + end + + def many_then_foo + 100.times { |i| instance_variable_set(:"@v#{i}", i) } + @bar = 3 + @foo = 4 + end + + def foo = @foo + 1 + end + + O1 = C.new + O1.foo_then_many + O2 = C.new + O2.many_then_foo + O1.foo + O1.foo + O1.foo + O1.foo + O2.foo + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:foo)"), @" + fn foo@<compiled>:15: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:CPtr = LoadField v11, :as_heap@0x1002 + v21:BasicObject = LoadField v20, :@foo@0x1000 + Jump bb4(v21) + bb6(): + v23:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v24:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v25 = RefineType v24, CUInt64 + v26:CInt64 = IntAnd v12, v23 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + v29:BasicObject = LoadField v11, :@foo@0x1004 + Jump bb4(v29) + bb8(): + v44:CShape = LoadField v11, :shape_id@0x1005 + v45:CShape[0x1006] = GuardBitEquals v44, CShape(0x1006) recompile + v46:CPtr = LoadField v11, :as_heap@0x1002 + v47:BasicObject = LoadField v46, :@foo@0x1000 + Jump bb4(v47) + bb4(v13:BasicObject): + v34:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v50:Fixnum = GuardType v13, Fixnum recompile + v51:Fixnum = FixnumAdd v50, v34 + CheckInterrupts + Return v51 + "); + } + + #[test] + fn test_optimize_getivar_polymorphic_with_subclass() { + set_call_threshold(3); + eval(r#" + class C + def initialize + @foo = 3 + end + + def foo = @foo + 1 + end + + class D < C + def initialize + super + @bar = 4 + end + end + + O1 = C.new + O2 = D.new + O1.foo + O2.foo + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:foo)"), @" + fn foo@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:BasicObject = LoadField v11, :@foo@0x1002 + Jump bb4(v20) + bb6(): + v22:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v23:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v24 = RefineType v23, CUInt64 + v25:CInt64 = IntAnd v12, v22 + v26:CBool = IsBitEqual v25, v24 + CondBranch v26, bb7(), bb8() + bb7(): + v28:BasicObject = LoadField v11, :@foo@0x1002 + Jump bb4(v28) + bb8(): + v30:BasicObject = GetIvar v11, :@foo + Jump bb4(v30) + bb4(v13:BasicObject): + v33:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v44:Fixnum = GuardType v13, Fixnum recompile + v45:Fixnum = FixnumAdd v44, v33 + CheckInterrupts + Return v45 + "); + } + + #[test] + fn test_getivar_polymorphic_t_class_and_t_data() { + set_call_threshold(3); + eval(r#" + module Reader + def test = @a + end + + class A + extend Reader + @a = 0 + end + + ARGF.instance_eval do + extend Reader + @a = :a + end + + A.test + ARGF.test + "#); + assert_snapshot!(assert_compiles("[A.test, ARGF.test]"), @"[0, :a]"); + assert_snapshot!(hir_string_proc("Reader.instance_method(:test)"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:HeapBasicObject = GuardType v6, HeapBasicObject + v12:CUInt64 = LoadField v11, :RBASIC_FLAGS@0x1000 + v14:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v15:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v16 = RefineType v15, CUInt64 + v17:CInt64 = IntAnd v12, v14 + v18:CBool = IsBitEqual v17, v16 + CondBranch v18, bb5(), bb6() + bb5(): + v20:RubyValue = LoadField v11, :fields_obj@0x1002 + v21:BasicObject = LoadField v20, :@a@0x1002 + Jump bb4(v21) + bb6(): + v23:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v24:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v25 = RefineType v24, CUInt64 + v26:CInt64 = IntAnd v12, v23 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb8() + bb7(): + PatchPoint RootBoxOnly + v30:RubyValue = LoadField v11, :fields_obj@0x1004 + v31:BasicObject = LoadField v30, :@a@0x1002 + Jump bb4(v31) + bb8(): + v33:BasicObject = GetIvar v11, :@a + Jump bb4(v33) + bb4(v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_dont_optimize_attr_accessor_polymorphic() { + set_call_threshold(3); + eval(" + class C + attr_reader :foo, :bar + + def foo_then_bar + @foo = 1 + @bar = 2 + end + + def bar_then_foo + @bar = 3 + @foo = 4 + end + end + + O1 = C.new + O1.foo_then_bar + O2 = C.new + O2.bar_then_foo + def test(o) = o.foo + test O1 + test O2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:20: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:ObjectSubclass[class_exact:C] = RefineType v18, ObjectSubclass[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v37:BasicObject = GetIvar v20, :@foo + Jump bb4(v16, v17, v37) + bb6(): + v24:BasicObject = Send v10, :foo # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v24) + bb4(v26:BasicObject, v27:BasicObject, v28:BasicObject): + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_optimize_getivar_with_complex_shape() { + eval(r#" + class C + attr_accessor :foo + end + obj = C.new + (0..1000).each do |i| + obj.instance_variable_set(:"@v#{i}", i) + end + (0..1000).each do |i| + obj.remove_instance_variable(:"@v#{i}") + end + def test(o) = o.foo + test obj + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v24:BasicObject = GetIvar v23, :@foo + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_optimize_send_with_block() { + eval(r#" + def test = [1, 2, 3].map { |x| x * 2 } + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, map@0x1010, cme:0x1018) + v22:BasicObject = SendDirect v11, 0x1040, :map (0x1050) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_optimize_send_variadic_with_block() { + eval(r#" + A = [1, 2, 3] + B = ["a", "b", "c"] + + def test + result = [] + A.zip(B) { |x, y| result << [x, y] } + result + end + + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v38:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, B) + v41:ArrayExact[VALUE(0x1018)] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(Array@0x1020) + PatchPoint MethodRedefined(Array@0x1020, zip@0x1028, cme:0x1030) + v46:BasicObject = CCallVariadic v38, :Array#zip@0x1058, v41 + PatchPoint NoEPEscape(test) + v24:CPtr = LoadSP + v25:BasicObject = LoadField v24, :result@0x1060 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_do_not_optimize_send_with_block_forwarding() { + eval(r#" + def test(&block) = [].map(&block) + test { |x| x }; test { |x| x } + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:ArrayExact = NewArray + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) + v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v26, v10) + bb6(v16:BasicObject, v17:BasicObject): + v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_replace_block_param_proxy_with_nil() { + eval(r#" + def test(&block) = [].map(&block) + test; test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:ArrayExact = NewArray + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[0] = GuardBitEquals v24, CInt64(0) + v26:NilClass = Const Value(nil) + Jump bb6(v26, v10) + bb6(v16:BasicObject, v17:BasicObject): + v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_replace_block_param_proxy_with_nil_nested() { + eval(r#" + def test(&block) + proc do + [].map(&block) + end + end + test; test + "#); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + v13:CPtr = GetEP 1 + v14:CUInt64 = LoadField v13, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v15:CBool = IsBlockParamModified v14 + CondBranch v15, bb4(), bb5() + bb4(): + v17:BasicObject = LoadField v13, :block@0x1001 + Jump bb6(v17) + bb5(): + v19:CInt64 = LoadField v13, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + v20:CInt64 = GuardAnyBitSet v19, CUInt64(1) + v21:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v21) + bb6(v12:BasicObject): + v24:BasicObject = Send v10, &block, :map, v12 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_send_direct_iseq_with_block_no_callee_block_param() { + let result = eval(r#" + def foo + yield 1 + end + + def test = foo { |x| x * 2 } + test; test + "#); + assert_eq!(VALUE::fixnum_from_usize(2), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_send_iseq_with_block_param_no_block() { + let result = eval(" + def foo(&blk) + blk ? blk.call : 42 + end + def test = foo + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(42), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v19:BasicObject = SendDirect v18, 0x1038, :foo (0x1048) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_send_bmethod_with_block_param_no_block() { + let result = eval(" + define_method(:foo) { |&blk| + blk ? blk.call : 42 + } + def test = foo + test + test + "); + assert_eq!(VALUE::fixnum_from_usize(42), result); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:BasicObject = SendDirect v19, 0x1038, :foo (0x1048) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_attr_reader_constant() { + eval(" + class C + attr_reader :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v26:CShape = LoadField v20, :shape_id@0x1048 + v27:CShape[0x1049] = GuardBitEquals v26, CShape(0x1049) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_accessor_constant() { + eval(" + class C + attr_accessor :foo + end + + O = C.new + def test = O.foo + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, foo@0x1018, cme:0x1020) + v26:CShape = LoadField v20, :shape_id@0x1048 + v27:CShape[0x1049] = GuardBitEquals v26, CShape(0x1049) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_reader() { + eval(" + class C + attr_reader :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_accessor() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:NilClass = Const Value(nil) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_attr_accessor_set() { + eval(" + class C + attr_accessor :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v31:CShape = LoadField v28, :shape_id@0x1040 + v32:CShape[0x1041] = GuardBitEquals v31, CShape(0x1041) + StoreField v28, :@foo@0x1042, v17 + WriteBarrier v28, v17 + v35:CShape[0x1043] = Const CShape(0x1043) + StoreField v28, :shape_id@0x1040, v35 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_inline_attr_writer_set() { + eval(" + class C + attr_writer :foo + end + + def test(o) = o.foo = 5 + test C.new + test C.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v31:CShape = LoadField v28, :shape_id@0x1040 + v32:CShape[0x1041] = GuardBitEquals v31, CShape(0x1041) + StoreField v28, :@foo@0x1042, v17 + WriteBarrier v28, v17 + v35:CShape[0x1043] = Const CShape(0x1043) + StoreField v28, :shape_id@0x1040, v35 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_inline_struct_aref_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v24:BasicObject = LoadField v23, :foo@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_inline_struct_aref_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) = o.foo + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v24:CPtr = LoadField v23, :as_heap@0x1040 + v25:BasicObject = LoadField v24, :foo@0x1041 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_elide_struct_aref() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o) + o.foo + 5 + end + test C.new + test C.new + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v27:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_inline_struct_aset_embedded() { + eval(r#" + C = Struct.new(:foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:BasicObject = LoadField v2, :v@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :o@1 + v9:BasicObject = LoadArg :v@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v31:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] recompile + v32:CUInt64 = LoadField v31, :RBASIC_FLAGS@0x1040 + v33:CUInt64 = GuardNoBitsSet v32, RUBY_FL_FREEZE=CUInt64(2048) + StoreField v31, :foo=@0x1041, v13 + WriteBarrier v31, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_inline_struct_aset_heap() { + eval(r#" + C = Struct.new(*(0..1000).map {|i| :"a#{i}"}, :foo) + def test(o, v) = o.foo = v + value = Object.new + test C.new, value + test C.new, value + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:BasicObject = LoadField v2, :v@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :o@1 + v9:BasicObject = LoadArg :v@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo=@0x1010, cme:0x1018) + v31:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] recompile + v32:CUInt64 = LoadField v31, :RBASIC_FLAGS@0x1040 + v33:CUInt64 = GuardNoBitsSet v32, RUBY_FL_FREEZE=CUInt64(2048) + v34:CPtr = LoadField v31, :as_heap@0x1041 + StoreField v34, :foo=@0x1042, v13 + WriteBarrier v31, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_array_reverse_returns_array() { + eval(r#" + def test = [].reverse + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v21:ArrayExact = CCallWithFrame v10, :Array#reverse@0x1038 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_array_reverse_is_elidable() { + eval(r#" + def test + [].reverse + 5 + end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + PatchPoint NoSingletonClass(Array@0x1000) + PatchPoint MethodRedefined(Array@0x1000, reverse@0x1008, cme:0x1010) + v16:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_array_join_returns_string() { + eval(r#" + def test = [].join "," + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:StringExact = StringCopy v12 + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, join@0x1010, cme:0x1018) + v24:StringExact = CCallVariadic v10, :Array#join@0x1040, v13 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_string_to_s_returns_string() { + eval(r#" + def test = "".to_s + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_string_literal_to_s() { + eval(r#" + def test = "foo".to_s + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_profiled_string_to_s() { + eval(r#" + def test(o) = o.to_s + test "foo" + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, to_s@0x1010, cme:0x1018) + v24:StringExact = GuardType v10, StringExact recompile + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fixnum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test 5 + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v23:Fixnum = GuardType v10, Fixnum recompile + v24:StringExact = CCallVariadic v23, :Integer#to_s@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_bignum_to_s_returns_string() { + eval(r#" + def test(x) = x.to_s + test (2**65) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v23:Bignum = GuardType v10, Bignum recompile + v24:StringExact = CCallVariadic v23, :Integer#to_s@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_fold_any_to_string_with_known_string_exact() { + eval(r##" + def test(x) = "#{x}" + test 123 + "##); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v28:Fixnum = GuardType v10, Fixnum + PatchPoint MethodRedefined(Integer@0x1010, to_s@0x1018, cme:0x1020) + v33:StringExact = CCallVariadic v28, :Integer#to_s@0x1048 + v22:StringExact = StringConcat v14, v33 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_array_aref_fixnum_literal() { + eval(" + def test + arr = [1, 2, 3] + arr[0] + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:ArrayExact = ArrayDup v13 + v19:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v38:CInt64[0] = Const CInt64(0) + v32:CInt64 = ArrayLength v14 + v33:CInt64[0] = GuardLess v38, v32 + v37:BasicObject = ArrayAref v14, v33 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_array_aref_fixnum_profiled() { + eval(" + def test(arr, idx) + arr[idx] + end + test([1, 2, 3], 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :arr@1 + v9:BasicObject = LoadArg :idx@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v29:ArrayExact = GuardType v12, ArrayExact recompile + v30:Fixnum = GuardType v13, Fixnum + v31:CInt64 = UnboxFixnum v30 + v32:CInt64 = ArrayLength v29 + v33:CInt64 = GuardLess v31, v32 + v34:CInt64 = AdjustBounds v33, v32 + v35:CInt64[0] = Const CInt64(0) + v36:CInt64 = GuardGreaterEq v34, v35 + v37:BasicObject = ArrayAref v29, v36 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_array_aref_fixnum_array_subclass() { + eval(" + class C < Array; end + def test(arr, idx) + arr[idx] + end + test(C.new([1, 2, 3]), 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :arr@1 + v9:BasicObject = LoadArg :idx@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, []@0x1010, cme:0x1018) + v29:ArraySubclass[class_exact:C] = GuardType v12, ArraySubclass[class_exact:C] recompile + v30:Fixnum = GuardType v13, Fixnum + v31:CInt64 = UnboxFixnum v30 + v32:CInt64 = ArrayLength v29 + v33:CInt64 = GuardLess v31, v32 + v34:CInt64 = AdjustBounds v33, v32 + v35:CInt64[0] = Const CInt64(0) + v36:CInt64 = GuardGreaterEq v34, v35 + v37:BasicObject = ArrayAref v29, v36 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_hash_aref_literal() { + eval(" + def test + arr = {1 => 3} + arr[1] + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:HashExact = HashDup v13 + v19:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v31:BasicObject = HashAref v14, v19 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_hash_aref_profiled() { + eval(" + def test(hash, key) + hash[key] + end + test({1 => 3}, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :hash@1 + v9:BasicObject = LoadArg :key@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []@0x1010, cme:0x1018) + v29:HashExact = GuardType v12, HashExact recompile + v30:BasicObject = HashAref v29, v13 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_no_optimize_hash_aref_subclass() { + eval(" + class C < Hash; end + def test(hash, key) + hash[key] + end + test(C.new({0 => 3}), 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :hash@1 + v9:BasicObject = LoadArg :key@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, []@0x1010, cme:0x1018) + v29:HashSubclass[class_exact:C] = GuardType v12, HashSubclass[class_exact:C] recompile + v30:BasicObject = CCallWithFrame v29, :Hash#[]@0x1040, v13 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_does_not_fold_hash_aref_with_frozen_hash() { + eval(" + H = {a: 0}.freeze + def test = H[:a] + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, H) + v23:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:StaticSymbol[:a] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(Hash@0x1018) + PatchPoint MethodRedefined(Hash@0x1018, []@0x1020, cme:0x1028) + v28:BasicObject = HashAref v23, v12 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_hash_aset_literal() { + eval(" + def test + h = {} + h[1] = 3 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:HashExact = NewHash + PatchPoint NoEPEscape(test) + v22:Fixnum[1] = Const Value(1) + v24:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Hash@0x1000) + PatchPoint MethodRedefined(Hash@0x1000, []=@0x1008, cme:0x1010) + HashAset v13, v22, v24 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_hash_aset_profiled() { + eval(" + def test(hash, key, val) + hash[key] = val + end + test({}, 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :hash@1 + v10:BasicObject = LoadArg :key@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, []=@0x1010, cme:0x1018) + v37:HashExact = GuardType v14, HashExact recompile + HashAset v37, v15, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_no_optimize_hash_aset_subclass() { + eval(" + class C < Hash; end + def test(hash, key, val) + hash[key] = val + end + test(C.new, 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + v4:BasicObject = LoadField v2, :key@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :hash@1 + v10:BasicObject = LoadArg :key@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, []=@0x1010, cme:0x1018) + v37:HashSubclass[class_exact:C] = GuardType v14, HashSubclass[class_exact:C] recompile + v38:BasicObject = CCallWithFrame v37, :Hash#[]=@0x1040, v15, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_optimize_thread_current() { + eval(" + def test = Thread.current + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Thread) + v20:ClassSubclass[Thread@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, current@0x1018, cme:0x1020) + v24:CPtr = LoadEC + v25:CPtr = LoadField v24, :thread_ptr@0x1048 + v26:BasicObject = LoadField v25, :self@0x1049 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_optimize_array_aset_literal() { + eval(" + def test(arr) + arr[1] = 10 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:Fixnum[1] = Const Value(1) + v19:Fixnum[10] = Const Value(10) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []=@0x1010, cme:0x1018) + v33:ArrayExact = GuardType v10, ArrayExact recompile + v34:CUInt64 = LoadField v33, :RBASIC_FLAGS@0x1040 + v35:CUInt64 = GuardNoBitsSet v34, RUBY_FL_FREEZE=CUInt64(2048) + v37:CUInt64 = GuardNoBitsSet v35, RUBY_ELTS_SHARED=CUInt64(4096) + v46:CInt64[1] = Const CInt64(1) + v39:CInt64 = ArrayLength v33 + v40:CInt64[1] = GuardLess v46, v39 + ArrayAset v33, v40, v19 + WriteBarrier v33, v19 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_optimize_array_aset_profiled() { + eval(" + def test(arr, index, val) + arr[index] = val + end + test([], 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :index@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :arr@1 + v10:BasicObject = LoadArg :index@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, []=@0x1010, cme:0x1018) + v37:ArrayExact = GuardType v14, ArrayExact recompile + v38:Fixnum = GuardType v15, Fixnum + v39:CUInt64 = LoadField v37, :RBASIC_FLAGS@0x1040 + v40:CUInt64 = GuardNoBitsSet v39, RUBY_FL_FREEZE=CUInt64(2048) + v42:CUInt64 = GuardNoBitsSet v40, RUBY_ELTS_SHARED=CUInt64(4096) + v43:CInt64 = UnboxFixnum v38 + v44:CInt64 = ArrayLength v37 + v45:CInt64 = GuardLess v43, v44 + v46:CInt64 = AdjustBounds v45, v44 + v47:CInt64[0] = Const CInt64(0) + v48:CInt64 = GuardGreaterEq v46, v47 + ArrayAset v37, v48, v16 + WriteBarrier v37, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_optimize_array_aset_array_subclass() { + eval(" + class MyArray < Array; end + def test(arr, index, val) + arr[index] = val + end + a = MyArray.new + test(a, 0, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + v4:BasicObject = LoadField v2, :index@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :arr@1 + v10:BasicObject = LoadArg :index@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(MyArray@0x1008) + PatchPoint MethodRedefined(MyArray@0x1008, []=@0x1010, cme:0x1018) + v37:ArraySubclass[class_exact:MyArray] = GuardType v14, ArraySubclass[class_exact:MyArray] recompile + v38:BasicObject = CCallVariadic v37, :Array#[]=@0x1040, v15, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_optimize_array_ltlt() { + eval(" + def test(arr) + arr << 1 + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, <<@0x1010, cme:0x1018) + v27:ArrayExact = GuardType v10, ArrayExact recompile + v28:CUInt64 = LoadField v27, :RBASIC_FLAGS@0x1040 + v29:CUInt64 = GuardNoBitsSet v28, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v27, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_array_push_single_arg() { + eval(" + def test(arr) + arr.push(1) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, push@0x1010, cme:0x1018) + v26:ArrayExact = GuardType v10, ArrayExact recompile + v27:CUInt64 = LoadField v26, :RBASIC_FLAGS@0x1040 + v28:CUInt64 = GuardNoBitsSet v27, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v26, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_do_not_optimize_array_push_multi_arg() { + eval(" + def test(arr) + arr.push(1,2,3) + end + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v17:Fixnum[2] = Const Value(2) + v19:Fixnum[3] = Const Value(3) + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, push@0x1010, cme:0x1018) + v30:ArrayExact = GuardType v10, ArrayExact recompile + v31:BasicObject = CCallVariadic v30, :Array#push@0x1040, v15, v17, v19 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_array_push_with_array_subclass() { + eval(" + class PushSubArray < Array + def <<(val) = super + end + test = PushSubArray.new + test << 1 + "); + assert_snapshot!(hir_string_proc("PushSubArray.new.method(:<<)"), @" + fn <<@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :val@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :val@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Array@0x1008, <<@0x1010, cme:0x1018) + v23:CPtr = GetEP 0 + v24:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v25:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v24, Value(VALUE(0x1048)) + v26:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v27:FalseClass = GuardBitEquals v26, Value(false) + v28:Array = GuardType v9, Array + v29:CUInt64 = LoadField v28, :RBASIC_FLAGS@0x1051 + v30:CUInt64 = GuardNoBitsSet v29, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v28, v10 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_array_pop_with_array_subclass() { + eval(" + class PopSubArray < Array + def pop = super + end + test = PopSubArray.new([1]) + test.pop + "); + assert_snapshot!(hir_string_proc("PopSubArray.new.method(:pop)"), @" + fn pop@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Array@0x1000, pop@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:Array = GuardType v6, Array + v24:CUInt64 = LoadField v23, :RBASIC_FLAGS@0x1049 + v25:CUInt64 = GuardNoBitsSet v24, RUBY_FL_FREEZE=CUInt64(2048) + v27:CUInt64 = GuardNoBitsSet v25, RUBY_ELTS_SHARED=CUInt64(4096) + v28:BasicObject = ArrayPop v23 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_array_aref_with_array_subclass_and_fixnum() { + eval(" + class ArefSubArray < Array + def [](idx) = super + end + test = ArefSubArray.new([1]) + test[0] + "); + assert_snapshot!(hir_string_proc("ArefSubArray.new.method(:[])"), @" + fn []@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :idx@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :idx@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v23:CPtr = GetEP 0 + v24:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v25:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v24, Value(VALUE(0x1048)) + v26:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v27:FalseClass = GuardBitEquals v26, Value(false) + v28:Array = GuardType v9, Array + v29:Fixnum = GuardType v10, Fixnum + v30:CInt64 = UnboxFixnum v29 + v31:CInt64 = ArrayLength v28 + v32:CInt64 = GuardLess v30, v31 + v33:CInt64 = AdjustBounds v32, v31 + v34:CInt64[0] = Const CInt64(0) + v35:CInt64 = GuardGreaterEq v33, v34 + v36:BasicObject = ArrayAref v28, v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_dont_optimize_array_aref_with_array_subclass_and_non_fixnum() { + eval(" + class ArefSubArrayRange < Array + def [](idx) = super + end + test = ArefSubArrayRange.new([1, 2, 3]) + test[0..1] + "); + assert_snapshot!(hir_string_proc("ArefSubArrayRange.new.method(:[])"), @" + fn []@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :idx@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :idx@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) + v23:CPtr = GetEP 0 + v24:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v25:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v24, Value(VALUE(0x1048)) + v26:RubyValue = LoadField v23, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v27:FalseClass = GuardBitEquals v26, Value(false) + v28:BasicObject = CCallVariadic v9, :Array#[]@0x1058, v10 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_array_length() { + eval(" + def test(arr) = arr.length + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, length@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact recompile + v26:CInt64 = ArrayLength v25 + v27:Fixnum = BoxFixnum v26 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_array_size() { + eval(" + def test(arr) = arr.size + test([]) + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arr@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :arr@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Array@0x1008) + PatchPoint MethodRedefined(Array@0x1008, size@0x1010, cme:0x1018) + v25:ArrayExact = GuardType v10, ArrayExact recompile + v26:CInt64 = ArrayLength v25 + v27:Fixnum = BoxFixnum v26 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_optimize_regexpmatch2() { + eval(r#" + def test(s) = s =~ /a/ + test("foo") + "#); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:RegexpExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, =~@0x1018, cme:0x1020) + v27:StringExact = GuardType v10, StringExact recompile + v28:BasicObject = CCallWithFrame v27, :String#=~@0x1048, v15 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_string_getbyte_fixnum() { + eval(r#" + def test(s, i) = s.getbyte(i) + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :i@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :i@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, getbyte@0x1010, cme:0x1018) + v28:StringExact = GuardType v12, StringExact recompile + v29:Fixnum = GuardType v13, Fixnum + v30:CInt64 = UnboxFixnum v29 + v31:CInt64 = LoadField v28, :len@0x1040 + v32:CInt64 = GuardLess v30, v31 + v33:CInt64 = AdjustBounds v32, v31 + v34:CInt64[0] = Const CInt64(0) + v35:CInt64 = GuardGreaterEq v33, v34 + v36:Fixnum = StringGetbyte v28, v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_elide_string_getbyte_fixnum() { + eval(r#" + def test(s, i) + s.getbyte(i) + 5 + end + test("foo", 0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :i@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :i@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, getbyte@0x1010, cme:0x1018) + v32:StringExact = GuardType v12, StringExact recompile + v33:Fixnum = GuardType v13, Fixnum + v34:CInt64 = UnboxFixnum v33 + v35:CInt64 = LoadField v32, :len@0x1040 + v36:CInt64 = GuardLess v34, v35 + v37:CInt64 = AdjustBounds v36, v35 + v38:CInt64[0] = Const CInt64(0) + v39:CInt64 = GuardGreaterEq v37, v38 + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_optimize_string_setbyte_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 127) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :s@1 + v10:BasicObject = LoadArg :idx@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, setbyte@0x1010, cme:0x1018) + v32:StringExact = GuardType v14, StringExact recompile + v33:Fixnum = GuardType v15, Fixnum + v34:Fixnum = GuardType v16, Fixnum + v35:CInt64 = UnboxFixnum v33 + v36:CInt64 = LoadField v32, :len@0x1040 + v37:CInt64 = GuardLess v35, v36 + v38:CInt64 = AdjustBounds v37, v36 + v39:CInt64[0] = Const CInt64(0) + v40:CInt64 = GuardGreaterEq v38, v39 + v41:CUInt64 = LoadField v32, :RBASIC_FLAGS@0x1041 + v42:CUInt64 = GuardNoBitsSet v41, RUBY_FL_FREEZE=CUInt64(2048) + v43:Fixnum = StringSetbyteFixnum v32, v33, v34 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_optimize_string_subclass_setbyte_fixnum() { + eval(r#" + class MyString < String + end + def test(s, idx, val) + s.setbyte(idx, val) + end + test(MyString.new('foo'), 0, 127) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :s@1 + v10:BasicObject = LoadArg :idx@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1008) + PatchPoint MethodRedefined(MyString@0x1008, setbyte@0x1010, cme:0x1018) + v32:StringSubclass[class_exact:MyString] = GuardType v14, StringSubclass[class_exact:MyString] recompile + v33:Fixnum = GuardType v15, Fixnum + v34:Fixnum = GuardType v16, Fixnum + v35:CInt64 = UnboxFixnum v33 + v36:CInt64 = LoadField v32, :len@0x1040 + v37:CInt64 = GuardLess v35, v36 + v38:CInt64 = AdjustBounds v37, v36 + v39:CInt64[0] = Const CInt64(0) + v40:CInt64 = GuardGreaterEq v38, v39 + v41:CUInt64 = LoadField v32, :RBASIC_FLAGS@0x1041 + v42:CUInt64 = GuardNoBitsSet v41, RUBY_FL_FREEZE=CUInt64(2048) + v43:Fixnum = StringSetbyteFixnum v32, v33, v34 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_do_not_optimize_string_setbyte_non_fixnum() { + eval(r#" + def test(s, idx, val) + s.setbyte(idx, val) + end + test("foo", 0, 3.14) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :idx@0x1001 + v5:BasicObject = LoadField v2, :val@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :s@1 + v10:BasicObject = LoadArg :idx@2 + v11:BasicObject = LoadArg :val@3 + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:BasicObject, v16:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, setbyte@0x1010, cme:0x1018) + v32:StringExact = GuardType v14, StringExact recompile + v33:BasicObject = CCallWithFrame v32, :String#setbyte@0x1040, v15, v16 + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_specialize_string_empty() { + eval(r#" + def test(s) + s.empty? + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, empty?@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact recompile + v26:CInt64 = LoadField v25, :len@0x1040 + v27:CInt64[0] = Const CInt64(0) + v28:CBool = IsBitEqual v26, v27 + v29:BoolExact = BoxBool v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_eliminate_string_empty() { + eval(r#" + def test(s) + s.empty? + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, empty?@0x1010, cme:0x1018) + v29:StringExact = GuardType v10, StringExact recompile + v20:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_integer_succ_with_fixnum() { + eval(" + def test(x) = x.succ + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, succ@0x1010, cme:0x1018) + v24:Fixnum = GuardType v10, Fixnum recompile + v25:Fixnum[1] = Const Value(1) + v26:Fixnum = FixnumAdd v24, v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_bignum() { + eval(" + def test(x) = x.succ + test(4 << 70) + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, succ@0x1010, cme:0x1018) + v24:Bignum = GuardType v10, Bignum recompile + v25:BasicObject = CCallWithFrame v24, :Integer#succ@0x1040 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_inline_integer_ltlt_with_known_fixnum() { + eval(" + def test(x) = x << 5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:Fixnum = FixnumLShift v26, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_negative() { + eval(" + def test(x) = x << -5 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:BasicObject = CCallWithFrame v26, :Integer#<<@0x1040, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_out_of_range() { + eval(" + def test(x) = x << 64 + test(4) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v26:Fixnum = GuardType v10, Fixnum recompile + v27:BasicObject = CCallWithFrame v26, :Integer#<<@0x1040, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_dont_inline_integer_ltlt_with_unknown_fixnum() { + eval(" + def test(x, y) = x << y + test(4, 5) + "); + assert_contains_opcode("test", YARVINSN_opt_ltlt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, <<@0x1010, cme:0x1018) + v28:Fixnum = GuardType v12, Fixnum recompile + v29:BasicObject = CCallWithFrame v28, :Integer#<<@0x1040, v13 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_inline_integer_gtgt_with_known_fixnum() { + eval(" + def test(x) = x >> 5 + test(4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v25:Fixnum = GuardType v10, Fixnum recompile + v26:Fixnum = FixnumRShift v25, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_negative() { + eval(" + def test(x) = x >> -5 + test(4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[-5] = Const Value(-5) + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v25:Fixnum = GuardType v10, Fixnum recompile + v26:BasicObject = CCallWithFrame v25, :Integer#>>@0x1040, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_out_of_range() { + eval(" + def test(x) = x >> 64 + test(4) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[64] = Const Value(64) + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v25:Fixnum = GuardType v10, Fixnum recompile + v26:BasicObject = CCallWithFrame v25, :Integer#>>@0x1040, v15 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_dont_inline_integer_gtgt_with_unknown_fixnum() { + eval(" + def test(x, y) = x >> y + test(4, 5) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, >>@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum recompile + v28:BasicObject = CCallWithFrame v27, :Integer#>>@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_optimize_string_append() { + eval(r#" + def test(x, y) = x << y + test("iron", "fish") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact recompile + v30:String = GuardType v13, String + v31:StringExact = StringAppend v29, v30 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_string_append_codepoint() { + eval(r#" + def test(x, y) = x << y + test("iron", 4) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact recompile + v30:Fixnum = GuardType v13, Fixnum + v31:StringExact = StringAppendCodepoint v29, v30 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_string_append_string_subclass() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test("iron", MyString.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact recompile + v30:String = GuardType v13, String + v31:StringExact = StringAppend v29, v30 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_do_not_optimize_string_subclass_append_string() { + eval(r#" + class MyString < String + end + def test(x, y) = x << y + test(MyString.new, "iron") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(MyString@0x1008) + PatchPoint MethodRedefined(MyString@0x1008, <<@0x1010, cme:0x1018) + v29:StringSubclass[class_exact:MyString] = GuardType v12, StringSubclass[class_exact:MyString] recompile + v30:BasicObject = CCallWithFrame v29, :String#<<@0x1040, v13 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_optimize_string_append_non_string() { + eval(r#" + def test = "iron" << :a + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, <<@0x1018, cme:0x1020) + v25:BasicObject = CCallWithFrame v11, :String#<<@0x1048, v13 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_many_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo(3, 3, 3) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:Fixnum[3] = Const Value(3) + v14:Fixnum[3] = Const Value(3) + v16:Fixnum[3] = Const Value(3) + v18:BasicObject = Send v10, :foo, v12, v14, v16 # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_optimize_string_ascii_only_p() { + eval(r#" + def test(x) = x.ascii_only? + test("iron") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ascii_only?@0x1010, cme:0x1018) + v24:StringExact = GuardType v10, StringExact recompile + v25:BoolExact = CCall v24, :String#ascii_only?@0x1040 + CheckInterrupts + Return v25 + "); + } + + #[test] + fn test_dont_optimize_when_passing_too_few_args() { + eval(r#" + public def foo(lead, opt=raise) = opt + def test = 0.foo + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[0] = Const Value(0) + v12:BasicObject = Send v10, :foo # SendFallbackReason: Argument count does not match parameter count + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_dont_inline_integer_succ_with_args() { + eval(" + def test = 4.succ 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[4] = Const Value(4) + v12:Fixnum[1] = Const Value(1) + v14:BasicObject = Send v10, :succ, v12 # SendFallbackReason: SendWithoutBlock: unsupported method type Cfunc + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_inline_integer_xor_with_fixnum() { + eval(" + def test(x, y) = x ^ y + test(1, 2) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum recompile + v28:Fixnum = GuardType v13, Fixnum + v29:Fixnum = FixnumXor v27, v28 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_eliminate_integer_xor() { + eval(r#" + def test(x, y) + x ^ y + 42 + end + test(1, 2) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v31:Fixnum = GuardType v12, Fixnum recompile + v32:Fixnum = GuardType v13, Fixnum + v23:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_bignum_lhs() { + eval(" + def test(x, y) = x ^ y + test(4 << 70, 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v27:Bignum = GuardType v12, Bignum recompile + v28:BasicObject = CCallWithFrame v27, :Integer#^@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_bignum_rhs() { + eval(" + def test(x, y) = x ^ y + test(1, 4 << 70) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, ^@0x1010, cme:0x1018) + v27:Fixnum = GuardType v12, Fixnum recompile + v28:BasicObject = CCallWithFrame v27, :Integer#^@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_boolean() { + eval(" + def test(x, y) = x ^ y + test(true, 0) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(TrueClass@0x1008, ^@0x1010, cme:0x1018) + v27:TrueClass = GuardType v12, TrueClass recompile + v28:BasicObject = CCallWithFrame v27, :TrueClass#^@0x1040, v13 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_dont_inline_integer_xor_with_args() { + eval(" + def test(x, y) = x.^() + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + SideExit NoProfileSend recompile + "); + } + + #[test] + fn test_specialize_hash_size() { + eval(" + def test(hash) = hash.size + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :hash@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, size@0x1010, cme:0x1018) + v25:HashExact = GuardType v10, HashExact recompile + v26:Fixnum = CCall v25, :Hash#size@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_eliminate_hash_size() { + eval(" + def test(hash) + hash.size + 5 + end + test({foo: 3, bar: 1, baz: 4}) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :hash@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :hash@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1008) + PatchPoint MethodRedefined(Hash@0x1008, size@0x1010, cme:0x1018) + v29:HashExact = GuardType v10, HashExact recompile + v20:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_optimize_respond_to_p_true() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v30:TrueClass = Const Value(true) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_no_method() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, respond_to_missing?@0x1048, cme:0x1050) + PatchPoint MethodRedefined(C@0x1010, foo@0x1078, cme:0x1080) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_default_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v30:FalseClass = Const Value(false) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_respond_to_p_false_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, false) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:FalseClass = Const Value(false) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_true_private() { + eval(r#" + class C + private + def foo; end + end + def test(o) = o.respond_to?(:foo, true) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:TrueClass = Const Value(true) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_truthy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, 4) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:Fixnum[4] = Const Value(4) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_p_falsy() { + eval(r#" + class C + def foo; end + end + def test(o) = o.respond_to?(:foo, nil) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + v17:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v28:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, foo@0x1048, cme:0x1050) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_optimize_respond_to_missing() { + eval(r#" + class C + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + PatchPoint MethodRedefined(C@0x1010, respond_to_missing?@0x1048, cme:0x1050) + PatchPoint MethodRedefined(C@0x1010, foo@0x1078, cme:0x1080) + v32:FalseClass = Const Value(false) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_do_not_optimize_redefined_respond_to_missing() { + eval(r#" + class C + def respond_to_missing?(method, include_private = false) + true + end + end + def test(o) = o.respond_to?(:foo) + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:7: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:StaticSymbol[:foo] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(C@0x1010) + PatchPoint MethodRedefined(C@0x1010, respond_to?@0x1018, cme:0x1020) + v26:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v27:BasicObject = CCallVariadic v26, :Kernel#respond_to?@0x1048, v15 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putself() { + eval(r#" + def callee = self + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_string() { + eval(r#" + # frozen_string_literal: true + def callee = "abc" + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:StringExact[VALUE(0x1038)] = Const Value(VALUE(0x1038)) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putnil() { + eval(r#" + def callee = nil + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:NilClass = Const Value(nil) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_true() { + eval(r#" + def callee = true + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:TrueClass = Const Value(true) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_false() { + eval(r#" + def callee = false + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:FalseClass = Const Value(false) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_zero() { + eval(r#" + def callee = 0 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:Fixnum[0] = Const Value(0) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_putobject_one() { + eval(r#" + def callee = 1 + def test = callee + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v20:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_send_without_block_direct_parameter() { + eval(r#" + def callee(x) = x + def test = callee 3 + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_without_block_direct_last_parameter() { + eval(r#" + def callee(x, y, z) = z + def test = callee 1, 2, 3 + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v15:Fixnum[3] = Const Value(3) + PatchPoint MethodRedefined(Object@0x1000, callee@0x1008, cme:0x1010) + v24:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_splat() { + eval(" + def foo = itself + + def test + # Use a local to inhibit compile.c peephole optimization to ensure callsites have VM_CALL_ARGS_SPLAT + empty = [] + foo(*empty) + ''.display(*empty) + itself(*empty) + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:ArrayExact = NewArray + v19:ArrayExact = ToArray v13 + v21:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing + v25:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v26:StringExact = StringCopy v25 + PatchPoint NoEPEscape(test) + v31:ArrayExact = ToArray v13 + v33:BasicObject = Send v26, :display, v31 # SendFallbackReason: Complex argument passing + PatchPoint NoEPEscape(test) + v41:ArrayExact = ToArray v13 + v43:BasicObject = Send v8, :itself, v41 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_inline_symbol_to_sym() { + eval(r#" + def test(o) = o.to_sym + test :foo + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1008, to_sym@0x1010, cme:0x1018) + v22:StaticSymbol = GuardType v10, StaticSymbol recompile + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_inline_integer_to_i() { + eval(r#" + def test(o) = o.to_i + test 5 + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, to_i@0x1010, cme:0x1018) + v22:Fixnum = GuardType v10, Fixnum recompile + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_inline_send_with_block_with_no_params() { + // Passing a block to a method that doesn't use it falls back to the + // interpreter so that unused block warnings are properly emitted. + eval(r#" + def callee = 123 + def test + callee do + end + end + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :callee # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_with_block_with_one_param() { + // Passing a block to a method that doesn't use it falls back to the + // interpreter so that unused block warnings are properly emitted. + eval(r#" + def callee = 123 + def test + callee do |_| + end + end + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :callee # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_inline_send_with_block_with_multiple_params() { + // Passing a block to a method that doesn't use it falls back to the + // interpreter so that unused block warnings are properly emitted. + eval(r#" + def callee = 123 + def test + callee do |_a, _b| + end + end + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, 0x1000, :callee # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_no_inline_send_with_symbol_block() { + eval(r#" + def callee = 123 + public def the_block = 456 + def test + callee(&:the_block) + end + puts test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:StaticSymbol[:the_block] = Const Value(VALUE(0x1000)) + v13:BasicObject = Send v6, &block, :callee, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_profile_stack_skips_block_arg() { + // Regression test: profile_stack must skip the &block arg on the stack when mapping + // profiled operand types. Without the fix, the receiver type would be mapped to the + // wrong stack slot, causing resolve_receiver_type to return NoProfile. + // With the fix, the receiver type is correctly resolved and the send gets past type + // resolution to hit the ARGS_BLOCKARG guard (ComplexArgPass) instead of NoProfile. + eval(" + def test(&block) = [].map(&block) + test { |x| x }; test { |x| x } + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:ArrayExact = NewArray + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64 = GuardAnyBitSet v24, CUInt64(1) + v26:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v26, v10) + bb6(v16:BasicObject, v17:BasicObject): + v29:BasicObject = Send v14, &block, :map, v16 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_optimize_stringexact_eq_stringexact() { + eval(r#" + def test(l, r) = l == r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact recompile + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_string_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, ==@0x1010, cme:0x1018) + v29:StringSubclass[class_exact:C] = GuardType v12, StringSubclass[class_exact:C] recompile + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_stringexact_eq_string() { + eval(r#" + class C < String + end + def test(l, r) = l == r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact recompile + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_stringexact() { + eval(r#" + def test(l, r) = l === r + test("a", "b") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ===@0x1010, cme:0x1018) + v28:StringExact = GuardType v12, StringExact recompile + v29:String = GuardType v13, String + v30:BoolExact = StringEqual v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_string_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test(C.new("a"), C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, ===@0x1010, cme:0x1018) + v28:StringSubclass[class_exact:C] = GuardType v12, StringSubclass[class_exact:C] recompile + v29:String = GuardType v13, String + v30:BoolExact = StringEqual v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_optimize_stringexact_eqq_string() { + eval(r#" + class C < String + end + def test(l, r) = l === r + test("a", C.new("b")) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :l@0x1000 + v4:BasicObject = LoadField v2, :r@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :l@1 + v9:BasicObject = LoadArg :r@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ===@0x1010, cme:0x1018) + v28:StringExact = GuardType v12, StringExact recompile + v29:String = GuardType v13, String + v30:BoolExact = StringEqual v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_string_equal_same_operand_true() { + eval(r#" + def test(s) = s == s + test("x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v26:StringExact = GuardType v10, StringExact recompile + v29:TrueClass = Const Value(true) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_fold_string_eqq_same_operand_true() { + eval(r#" + def test(s) = s === s + test("x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ===@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact recompile + v28:TrueClass = Const Value(true) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_fold_string_equal_frozen_local_same_operand_true() { + eval(r#" + def test + str = "a".freeze + str == str + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v14:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v32:TrueClass = Const Value(true) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_fold_string_equal_frozen_distinct_literals_false() { + eval(r#" + def test + "a".freeze == "b".freeze + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v28:FalseClass = Const Value(false) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_not_fold_string_equal_true_without_pragma() { + eval(r#" + def test + "a" == "a" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v14:StringExact = StringCopy v13 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v27:BoolExact = StringEqual v11, v14 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_not_fold_string_equal_false_without_pragma() { + eval(r#" + def test + "a" == "b" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + v13:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v14:StringExact = StringCopy v13 + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v27:BoolExact = StringEqual v11, v14 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_string_equal_true_with_pragma() { + eval(r#" + # frozen_string_literal: true + def test + "a" == "a" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v26:TrueClass = Const Value(true) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_string_equal_false_with_pragma() { + eval(r#" + # frozen_string_literal: true + def test + "a" == "b" + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v26:FalseClass = Const Value(false) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_not_fold_string_equal_after_string_append_mutation() { + eval(r#" + def test + a = "a" + b = "a" + a << "a" + a == b + end + + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb3(v6, v7, v8) + bb3(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v17:StringExact = StringCopy v16 + v21:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v22:StringExact = StringCopy v21 + v27:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v28:StringExact = StringCopy v27 + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, <<@0x1010, cme:0x1018) + v50:StringExact = StringAppend v17, v28 + PatchPoint NoEPEscape(test) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1040, cme:0x1048) + v55:BoolExact = StringEqual v17, v22 + CheckInterrupts + Return v55 + "); + } + + #[test] + fn test_not_fold_string_equal_distinct_objects() { + eval(r#" + def test(s, t) = s == t + test("x", "x") + test("x", "x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :t@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :t@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, ==@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact recompile + v30:String = GuardType v13, String + v31:BoolExact = StringEqual v29, v30 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_not_fold_string_equal_one_side_known_literal() { + eval(r#" + def test(s) = "a" == s + test("a") + test("a") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v15:StringExact = StringCopy v14 + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, ==@0x1018, cme:0x1020) + v29:String = GuardType v10, String + v30:BoolExact = StringEqual v15, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn opt_neq_string_nil_falls_back_to_basic_object_neq() { + eval(r#" + def test(str) + str != nil + end + + test("x") + test("x") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :str@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :str@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:NilClass = Const Value(nil) + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, !=@0x1010, cme:0x1018) + v27:StringExact = GuardType v10, StringExact recompile + v28:BoolExact = CCallWithFrame v27, :BasicObject#!=@0x1040, v15 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_inline_string_not_equal_distinct_objects() { + eval(r#" + def test(s, t) = s != t + test("x", "x") + test("x", "x") + "#); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :t@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :t@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, !=@0x1010, cme:0x1018) + v29:StringExact = GuardType v12, StringExact recompile + PatchPoint MethodRedefined(String@0x1008, ==@0x1040, cme:0x1048) + v33:String = GuardType v13, String + v34:BoolExact = StringEqual v29, v33 + v35:TrueClass = Const Value(true) + v36:CBool = IsBitNotEqual v34, v35 + v37:BoolExact = BoxBool v36 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_fold_string_not_equal_same_operand_false() { + eval(r#" + def test(s) = s != s + test("x") + test("x") + "#); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, !=@0x1010, cme:0x1018) + v26:StringExact = GuardType v10, StringExact recompile + PatchPoint MethodRedefined(String@0x1008, ==@0x1040, cme:0x1048) + v35:TrueClass = Const Value(true) + v32:TrueClass = Const Value(true) + v33:CBool = IsBitNotEqual v35, v32 + v34:BoolExact = BoxBool v33 + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_specialize_string_size() { + eval(r#" + def test(s) + s.size + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, size@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact recompile + v26:Fixnum = CCall v25, :String#size@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_elide_string_size() { + eval(r#" + def test(s) + s.size + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, size@0x1010, cme:0x1018) + v29:StringExact = GuardType v10, StringExact recompile + v20:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_inline_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, bytesize@0x1010, cme:0x1018) + v24:StringExact = GuardType v10, StringExact recompile + v25:CInt64 = LoadField v24, :len@0x1040 + v26:Fixnum = BoxFixnum v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_elide_string_bytesize() { + eval(r#" + def test(s) + s.bytesize + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, bytesize@0x1010, cme:0x1018) + v28:StringExact = GuardType v10, StringExact recompile + v19:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_specialize_string_length() { + eval(r#" + def test(s) + s.length + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, length@0x1010, cme:0x1018) + v25:StringExact = GuardType v10, StringExact recompile + v26:Fixnum = CCall v25, :String#length@0x1040 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_specialize_class_eqq() { + eval(r#" + def test(o) = String === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, String) + v27:ClassSubclass[String@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoEPEscape(test) + PatchPoint MethodRedefined(Class@0x1018, ===@0x1020, cme:0x1028) + v31:BoolExact = IsA v10, v27 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_dont_specialize_module_eqq() { + eval(r#" + def test(o) = Kernel === o + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Kernel) + v27:ModuleSubclass[Kernel@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoEPEscape(test) + PatchPoint MethodRedefined(Module@0x1018, ===@0x1020, cme:0x1028) + v31:BoolExact = CCall v27, :Module#===@0x1050, v10 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_specialize_is_a_class() { + eval(r#" + def test(o) = o.is_a?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, String) + v25:ClassSubclass[String@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, is_a?@0x1011, cme:0x1018) + v29:StringExact = GuardType v10, StringExact recompile + v30:BoolExact = IsA v29, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_specialize_is_a_module() { + eval(r#" + def test(o) = o.is_a?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Kernel) + v25:ModuleSubclass[Kernel@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, is_a?@0x1020, cme:0x1028) + v29:StringExact = GuardType v10, StringExact recompile + v30:BasicObject = CCallWithFrame v29, :Kernel#is_a?@0x1050, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_elide_is_a() { + eval(r#" + def test(o) + o.is_a?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Integer) + v29:ClassSubclass[Integer@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, is_a?@0x1020, cme:0x1028) + v33:StringExact = GuardType v10, StringExact recompile + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_elide_class_eqq() { + eval(r#" + def test(o) + Integer === o + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Integer) + v31:ClassSubclass[Integer@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoEPEscape(test) + PatchPoint MethodRedefined(Class@0x1018, ===@0x1020, cme:0x1028) + v23:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_specialize_kind_of_class() { + eval(r#" + def test(o) = o.kind_of?(String) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, String) + v25:ClassSubclass[String@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, kind_of?@0x1011, cme:0x1018) + v29:StringExact = GuardType v10, StringExact recompile + v30:BoolExact = IsA v29, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_dont_specialize_kind_of_module() { + eval(r#" + def test(o) = o.kind_of?(Kernel) + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Kernel) + v25:ModuleSubclass[Kernel@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, kind_of?@0x1020, cme:0x1028) + v29:StringExact = GuardType v10, StringExact recompile + v30:BasicObject = CCallWithFrame v29, :Kernel#kind_of?@0x1050, v25 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_elide_kind_of() { + eval(r#" + def test(o) + o.kind_of?(Integer) + 5 + end + test("asdf") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1008, Integer) + v29:ClassSubclass[Integer@0x1010] = Const Value(VALUE(0x1010)) + PatchPoint NoSingletonClass(String@0x1018) + PatchPoint MethodRedefined(String@0x1018, kind_of?@0x1020, cme:0x1028) + v33:StringExact = GuardType v10, StringExact recompile + v21:Fixnum[5] = Const Value(5) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_fold_is_a_true() { + eval(r#" + def test = 5.is_a?(Integer) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Integer) + v22:ClassSubclass[Integer@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Integer@0x1008, is_a?@0x1009, cme:0x1010) + v27:TrueClass = Const Value(true) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_is_a_false() { + eval(r#" + def test = 5.is_a?(String) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, String) + v22:ClassSubclass[String@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Integer@0x1010, is_a?@0x1018, cme:0x1020) + v27:FalseClass = Const Value(false) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_is_a_array_subclass_folds_to_true() { + eval(r#" + class C < Array; end + O = C.new + def test = O.is_a?(Array) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v22:ArraySubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Array) + v25:ClassSubclass[Array@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(C@0x1020) + PatchPoint MethodRedefined(C@0x1020, is_a?@0x1028, cme:0x1030) + v31:TrueClass = Const Value(true) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_is_a_user_defined_class_folds_to_true() { + eval(r#" + class C; end + O = C.new + def test = O.is_a?(C) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v22:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, C) + v25:ClassSubclass[C@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint NoSingletonClass(C@0x1018) + PatchPoint MethodRedefined(C@0x1018, is_a?@0x1019, cme:0x1020) + v31:TrueClass = Const Value(true) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_is_a_symbol_folds_to_true() { + eval(r#" + O = :my_static_symbol + def test = O.is_a?(Symbol) + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, O) + v22:StaticSymbol[:my_static_symbol] = Const Value(VALUE(0x1008)) + PatchPoint StableConstantNames(0x1010, Symbol) + v25:ClassSubclass[Symbol@0x1018] = Const Value(VALUE(0x1018)) + PatchPoint MethodRedefined(Symbol@0x1018, is_a?@0x1019, cme:0x1020) + v30:TrueClass = Const Value(true) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn counting_complex_feature_use_for_fallback() { + eval(" + define_method(:fancy) { |_a, *_b, kw: 100, **kw_rest, &block| } + def test = fancy(1) + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[1] = Const Value(1) + v13:BasicObject = Send v6, :fancy, v11 # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v13 + "); + } + + #[test] + fn call_method_forwardable_param() { + eval(" + def forwardable(...) = itself(...) + def call_forwardable = forwardable + call_forwardable + "); + assert_snapshot!(hir_string("call_forwardable"), @" + fn call_forwardable@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = Send v6, :forwardable # SendFallbackReason: Complex argument passing + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_elide_string_length() { + eval(r#" + def test(s) + s.length + 4 + end + test("this should get removed") + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :s@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(String@0x1008) + PatchPoint MethodRedefined(String@0x1008, length@0x1010, cme:0x1018) + v29:StringExact = GuardType v10, StringExact recompile + v20:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_fold_self_class_respond_to_true() { + eval(r#" + class C + class << self + attr_accessor :_lex_actions + private :_lex_actions, :_lex_actions= + end + self._lex_actions = [1, 2, 3] + def initialize + if self.class.respond_to?(:_lex_actions, true) + :CORRECT + else + :oh_no_wrong + end + end + end + C.new # warm up + TEST = C.instance_method(:initialize) + "#); + assert_snapshot!(hir_string_proc("TEST"), @" + fn initialize@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, class@0x1008, cme:0x1010) + v43:ObjectSubclass[class_exact:C] = GuardType v6, ObjectSubclass[class_exact:C] recompile + v46:ClassSubclass[C@0x1000] = Const Value(VALUE(0x1000)) + v13:StaticSymbol[:_lex_actions] = Const Value(VALUE(0x1038)) + v15:TrueClass = Const Value(true) + PatchPoint MethodRedefined(Class@0x1040, respond_to?@0x1048, cme:0x1050) + PatchPoint MethodRedefined(Class@0x1040, _lex_actions@0x1078, cme:0x1080) + v52:TrueClass = Const Value(true) + CheckInterrupts + v26:StaticSymbol[:CORRECT] = Const Value(VALUE(0x10a8)) + Return v26 + "); + } + + #[test] + fn test_fold_self_class_name() { + eval(r#" + class C; end + def test(o) = o.class.name + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, class@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v28:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1040, name@0x1048, cme:0x1050) + v32:StringExact|NilClass = CCall v28, :Module#name@0x1078 + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_fold_kernel_class() { + eval(r#" + class C; end + def test(o) = o.class + test(C.new) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, class@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile + v26:ClassSubclass[C@0x1008] = Const Value(VALUE(0x1008)) + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_fold_fixnum_class() { + eval(r#" + def test = 5.class + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[5] = Const Value(5) + PatchPoint MethodRedefined(Integer@0x1000, class@0x1008, cme:0x1010) + v22:ClassSubclass[Integer@0x1000] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_fold_singleton_class() { + eval(r#" + def test = self.class + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Object@0x1000, class@0x1008, cme:0x1010) + v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + v21:ClassSubclass[Object@0x1038] = Const Value(VALUE(0x1038)) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_print_nil_module_name() { + eval(r#" + X = [Module.new].freeze + def test = X[0] + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, X) + v23:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, []@0x1018, cme:0x1020) + v37:ModuleExact[VALUE(0x1048)] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v37 + "); + } + + #[test] + fn no_load_from_ep_right_after_entrypoint() { + let formatted = eval(" + def read_nil_local(a, _b, _c) + formatted ||= a + @formatted = formatted + -> { formatted } # the environment escapes + end + + def call + puts [], [], [], [] # fill VM stack with junk + read_nil_local(true, 1, 1) # expected direct send + end + + call # profile + call # compile + @formatted + "); + assert_eq!(Qtrue, formatted, "{}", formatted.obj_info()); + assert_snapshot!(hir_string("read_nil_local"), @" + fn read_nil_local@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :_b@0x1001 + v5:BasicObject = LoadField v2, :_c@0x1002 + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :_b@2 + v12:BasicObject = LoadArg :_c@3 + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:NilClass): + CheckInterrupts + SetLocal :formatted, l0, EP@3, v16 + PatchPoint SingleRactorMode + SetIvar v15, :@formatted, v16 + v47:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(Class@0x1010, lambda@0x1018, cme:0x1020) + v63:BasicObject = CCallWithFrame v47, :RubyVM::FrozenCore.lambda@0x1048, block=0x1050 + v50:CPtr = GetEP 0 + v51:BasicObject = LoadField v50, :a@0x1001 + v52:BasicObject = LoadField v50, :_b@0x1002 + v53:BasicObject = LoadField v50, :_c@0x1058 + v54:BasicObject = LoadField v50, :formatted@0x1059 + CheckInterrupts + Return v63 + "); + } + + #[test] + fn test_fold_load_field_frozen_constant_object() { + // Basic case: frozen constant object with attr_accessor + eval(" + class TestFrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + FROZEN_OBJ = TestFrozen.new.freeze + + def test = FROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_OBJ) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozen@0x1010) + PatchPoint MethodRedefined(TestFrozen@0x1010, a@0x1018, cme:0x1020) + v30:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_multiple_ivars() { + // Frozen object with multiple instance variables + eval(" + class TestMultiIvars + attr_accessor :a, :b, :c + def initialize + @a = 10 + @b = 20 + @c = 30 + end + end + + MULTI_FROZEN = TestMultiIvars.new.freeze + + def test = MULTI_FROZEN.b + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:13: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, MULTI_FROZEN) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestMultiIvars@0x1010) + PatchPoint MethodRedefined(TestMultiIvars@0x1010, b@0x1018, cme:0x1020) + v30:Fixnum[20] = Const Value(20) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_string_value() { + // Frozen object with a string ivar + eval(r#" + class TestFrozenStr + attr_accessor :name + def initialize + @name = "hello" + end + end + + FROZEN_STR = TestFrozenStr.new.freeze + + def test = FROZEN_STR.name + test + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_STR) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenStr@0x1010) + PatchPoint MethodRedefined(TestFrozenStr@0x1010, name@0x1018, cme:0x1020) + v30:StringExact[VALUE(0x1048)] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_nil_value() { + // Frozen object with nil ivar + eval(" + class TestFrozenNil + attr_accessor :value + def initialize + @value = nil + end + end + + FROZEN_NIL = TestFrozenNil.new.freeze + + def test = FROZEN_NIL.value + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_NIL) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenNil@0x1010) + PatchPoint MethodRedefined(TestFrozenNil@0x1010, value@0x1018, cme:0x1020) + v30:NilClass = Const Value(nil) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_no_fold_load_field_unfrozen_object() { + // Non-frozen object should NOT be folded + eval(" + class TestUnfrozen + attr_accessor :a + def initialize + @a = 1 + end + end + + UNFROZEN_OBJ = TestUnfrozen.new + + def test = UNFROZEN_OBJ.a + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, UNFROZEN_OBJ) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestUnfrozen@0x1010) + PatchPoint MethodRedefined(TestUnfrozen@0x1010, a@0x1018, cme:0x1020) + v26:CShape = LoadField v20, :shape_id@0x1048 + v27:CShape[0x1049] = GuardBitEquals v26, CShape(0x1049) recompile + v28:BasicObject = LoadField v20, :@a@0x104a + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_fold_load_field_frozen_with_attr_reader() { + // Using attr_reader instead of attr_accessor + eval(" + class TestAttrReader + attr_reader :value + def initialize(v) + @value = v + end + end + + FROZEN_READER = TestAttrReader.new(42).freeze + + def test = FROZEN_READER.value + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_READER) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestAttrReader@0x1010) + PatchPoint MethodRedefined(TestAttrReader@0x1010, value@0x1018, cme:0x1020) + v30:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_symbol_value() { + // Frozen object with a symbol ivar + eval(" + class TestFrozenSym + attr_accessor :sym + def initialize + @sym = :hello + end + end + + FROZEN_SYM = TestFrozenSym.new.freeze + + def test = FROZEN_SYM.sym + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_SYM) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenSym@0x1010) + PatchPoint MethodRedefined(TestFrozenSym@0x1010, sym@0x1018, cme:0x1020) + v30:StaticSymbol[:hello] = Const Value(VALUE(0x1048)) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_fold_load_field_frozen_true_false() { + // Frozen object with boolean ivars + eval(" + class TestFrozenBool + attr_accessor :flag + def initialize + @flag = true + end + end + + FROZEN_TRUE = TestFrozenBool.new.freeze + + def test = FROZEN_TRUE.flag + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, FROZEN_TRUE) + v20:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestFrozenBool@0x1010) + PatchPoint MethodRedefined(TestFrozenBool@0x1010, flag@0x1018, cme:0x1020) + v30:TrueClass = Const Value(true) + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_no_fold_load_field_dynamic_receiver() { + // Dynamic receiver (not a constant) should NOT be folded even if object is frozen + eval(" + class TestDynamic + attr_accessor :val + def initialize + @val = 99 + end + end + + def test(obj) = obj.val + o = TestDynamic.new.freeze + test o + test o + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :obj@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint NoSingletonClass(TestDynamic@0x1008) + PatchPoint MethodRedefined(TestDynamic@0x1008, val@0x1010, cme:0x1018) + v23:ObjectSubclass[class_exact:TestDynamic] = GuardType v10, ObjectSubclass[class_exact:TestDynamic] recompile + v26:CShape = LoadField v23, :shape_id@0x1040 + v27:CShape[0x1041] = GuardBitEquals v26, CShape(0x1041) recompile + v28:BasicObject = LoadField v23, :@val@0x1042 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_fold_load_field_frozen_nested_access() { + // Accessing multiple fields from frozen constant in sequence + eval(" + class TestNestedAccess + attr_accessor :x, :y + def initialize + @x = 100 + @y = 200 + end + end + + NESTED_FROZEN = TestNestedAccess.new.freeze + + def test = NESTED_FROZEN.x + NESTED_FROZEN.y + test + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, NESTED_FROZEN) + v27:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(TestNestedAccess@0x1010) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, x@0x1018, cme:0x1020) + v53:Fixnum[100] = Const Value(100) + PatchPoint StableConstantNames(0x1048, NESTED_FROZEN) + v34:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint MethodRedefined(TestNestedAccess@0x1010, y@0x1050, cme:0x1058) + v55:Fixnum[200] = Const Value(200) + PatchPoint MethodRedefined(Integer@0x1080, +@0x1088, cme:0x1090) + v56:Fixnum[300] = Const Value(300) + CheckInterrupts + Return v56 + "); + } + + #[test] + fn test_dont_fold_load_field_with_primitive_return_type() { + eval(r#" + S = "abc".freeze + def test = S.bytesize + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, S) + v20:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(String@0x1010) + PatchPoint MethodRedefined(String@0x1010, bytesize@0x1018, cme:0x1020) + v25:CInt64 = LoadField v20, :len@0x1048 + v26:Fixnum = BoxFixnum v25 + CheckInterrupts + Return v26 + "); + } + + #[test] + fn optimize_call_to_private_method_iseq_with_fcall() { + eval(r#" + class C + def callprivate = secret + private def secret = 42 + end + C.new.callprivate + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprivate)"), @" + fn callprivate@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact:C] = GuardType v6, ObjectSubclass[class_exact:C] recompile + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_iseq() { + eval(r#" + class C + private def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + #[test] + fn optimize_call_to_private_method_cfunc_with_fcall() { + eval(r#" + class BasicObject + def callprivate = initialize rescue $! + end + Obj = BasicObject.new.callprivate + "#); + assert_snapshot!(hir_string_proc("BasicObject.instance_method(:callprivate)"), @" + fn callprivate@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(BasicObject@0x1000) + PatchPoint MethodRedefined(BasicObject@0x1000, initialize@0x1008, cme:0x1010) + v21:BasicObjectExact = GuardType v6, BasicObjectExact recompile + v22:NilClass = Const Value(nil) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn dont_optimize_call_to_private_method_cfunc() { + eval(r#" + Obj = BasicObject.new + def test = Obj.initialize rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:BasicObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :initialize # SendFallbackReason: Send: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + #[test] + fn dont_optimize_call_to_private_top_level_method() { + eval(r#" + def toplevel_method = :OK + Obj = Object.new + def test = Obj.toplevel_method rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :toplevel_method # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + #[test] + fn optimize_call_to_protected_method_iseq_with_fcall() { + eval(r#" + class C + def callprotected = secret + protected def secret = 42 + end + C.new.callprotected + "#); + assert_snapshot!(hir_string_proc("C.instance_method(:callprotected)"), @" + fn callprotected@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint NoSingletonClass(C@0x1000) + PatchPoint MethodRedefined(C@0x1000, secret@0x1008, cme:0x1010) + v19:ObjectSubclass[class_exact:C] = GuardType v6, ObjectSubclass[class_exact:C] recompile + v21:Fixnum[42] = Const Value(42) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn dont_optimize_call_to_protected_method_iseq() { + eval(r#" + class C + protected def secret = 42 + end + Obj = C.new + def test = Obj.secret rescue $! + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, Obj) + v21:ObjectSubclass[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v12:BasicObject = Send v21, :secret # SendFallbackReason: SendWithoutBlock: method private or protected and no FCALL + CheckInterrupts + Return v12 + "); + } + + // Test that when a singleton class has been seen for a class, we skip the + // NoSingletonClass optimization to avoid an invalidation loop. + #[test] + fn test_skip_optimization_after_singleton_class_seen() { + // First, compile a function that uses the NoSingletonClass assumption + eval(r#" + def test(s, proc) + s.length + proc.call + s.length + end + test("hi", -> {}) + test("hi", -> {}) + "#); + let hir = hir_string("test"); + assert!(hir.contains("NoSingletonClass(String"), "{hir}"); + + // Now we break the assumption by defining a singleton method on a string. + eval(r#" + special_string = +"" + test(special_string, -> { def special_string.length = -1 }) + "#); + + // The output should NOT have NoSingletonClass patchpoint for String, and should + // fall back to SendWithoutBlock instead of the optimized CCall path. + let hir = hir_string("test"); + assert!(! hir.contains("NoSingletonClass(String"), "{hir}"); + assert_snapshot!(hir, @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :s@0x1000 + v4:BasicObject = LoadField v2, :proc@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :s@1 + v9:BasicObject = LoadArg :proc@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:BasicObject = Send v12, :length # SendFallbackReason: Singleton class previously created for receiver class + PatchPoint NoSingletonClass(Proc@0x1008) + PatchPoint MethodRedefined(Proc@0x1008, call@0x1010, cme:0x1018) + v40:ObjectSubclass[class_exact:Proc] = GuardType v13, ObjectSubclass[class_exact:Proc] recompile + v41:BasicObject = InvokeProc v40 + PatchPoint NoEPEscape(test) + v32:BasicObject = Send v12, :length # SendFallbackReason: Singleton class previously created for receiver class + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_no_singleton_class_busts_isolated_per_iseq() { + // First, compile a function that uses the NoSingletonClass assumption + eval(r#" + def will_bust(s, proc) + s.length + proc.call + s.length + end + + def call_length(s) = s.length + + will_bust("hi", -> {}) + will_bust("hi", -> {}) + "#); + let hir = hir_string("will_bust"); + assert!(hir.contains("NoSingletonClass(String"), "{hir}"); + + // Now we break the assumption by defining a singleton method on a string. + eval(r#" + special_string = +"" + will_bust(special_string, -> { def special_string.length = -1 }) + "#); + let hir = hir_string("will_bust"); + assert!(! hir.contains("NoSingletonClas(String"), "{hir}"); + + // But, the unrelated call_length() should still use NoSingletonClass + eval("call_length('profile')"); + let hir = hir_string("call_length"); + assert!(hir.contains("NoSingletonClass"), "{hir}"); + } + + #[test] + fn test_invokesuper_to_iseq_optimizes_to_direct() { + eval(" + class A + def foo + 'A' + end + end + + class B < A + def foo + super + end + end + + B.new.foo; B.new.foo + "); + + // A Ruby method as the target of `super` should optimize provided no block is given. + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendDirect but got:\n{hir}"); + assert!(hir.contains("SendDirect"), "Should optimize to SendDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:HeapBasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:HeapBasicObject): + PatchPoint MethodRedefined(A@0x1000, foo@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:BasicObject = SendDirect v6, 0x1050, :foo (0x1060) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_invokesuper_from_a_block() { + _ = eval(" + define_method(:itself) { super() } + itself + "); + + assert_snapshot!(hir_string("itself"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: call from within a block + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_with_positional_args_optimizes_to_direct() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(x) + super(x) + 1 + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to SendDirect but got:\n{hir}"); + assert!(hir.contains("SendDirect"), "Should optimize to SendDirect for call without args or block:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:HeapBasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:HeapBasicObject, v10:BasicObject): + PatchPoint MethodRedefined(A@0x1008, foo@0x1010, cme:0x1018) + v28:CPtr = GetEP 0 + v29:RubyValue = LoadField v28, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v30:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v29, Value(VALUE(0x1048)) + v31:RubyValue = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v32:FalseClass = GuardBitEquals v31, Value(false) + v33:BasicObject = SendDirect v9, 0x1058, :foo (0x1068), v10 + v18:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1070, +@0x1078, cme:0x1080) + v36:Fixnum = GuardType v33, Fixnum recompile + v37:Fixnum = FixnumAdd v36, v18 + CheckInterrupts + Return v37 + "); + } + + #[test] + fn test_invokesuper_with_forwarded_splat_args_remains_invokesuper() { + eval(" + class A + def foo(x) + x * 2 + end + end + + class B < A + def foo(*x) + super + end + end + + B.new.foo(5); B.new.foo(5) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:HeapBasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:HeapBasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v18:BasicObject = InvokeSuper v9, 0x1008, v16 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_invokesuper_with_block_literal_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo + super { 'from subclass' } + end + end + + B.new.foo; B.new.foo + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for block literal:\n{hir}"); + + // With a block, we don't optimize to SendDirect + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:HeapBasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:HeapBasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: super: call made with a block + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_invokesuper_to_cfunc_optimizes_to_ccall() { + eval(" + class C < Hash + def size + super + end + end + + C.new.size + "); + + let hir = hir_string_proc("C.new.method(:size)"); + assert!(!hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + + assert_snapshot!(hir, @" + fn size@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(Hash@0x1000, size@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:Fixnum = CCall v6, :Hash#size@0x1050 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_inline_invokesuper_to_basicobject_initialize() { + eval(" + class C + def initialize + super + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint MethodRedefined(BasicObject@0x1000, initialize@0x1008, cme:0x1010) + v18:CPtr = GetEP 0 + v19:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_ME_CREF@0x1038 + v20:CallableMethodEntry[VALUE(0x1040)] = GuardBitEquals v19, Value(VALUE(0x1040)) + v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 + v22:FalseClass = GuardBitEquals v21, Value(false) + v23:NilClass = Const Value(nil) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_invokesuper_to_variadic_cfunc_optimizes_to_ccall() { + eval(" + class MyString < String + def byteindex(needle, offset = 0) + super(needle, offset) + end + end + + MyString.new('hello world').byteindex('world', 0); MyString.new('hello world').byteindex('world', 0) + "); + + let hir = hir_string_proc("MyString.new('hello world').method(:byteindex)"); + assert!(!hir.contains("InvokeSuper "), "InvokeSuper should optimize to CCallVariadic but got:\n{hir}"); + assert!(hir.contains("CCallVariadic"), "Should optimize to CCallVariadic for variadic cfunc:\n{hir}"); + + assert_snapshot!(hir, @" + fn byteindex@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :needle@0x1000 + v4:BasicObject = LoadField v2, :offset@0x1001 + v5:CPtr = LoadPC + v6:CPtr[CPtr(0x1002)] = Const CPtr(0x1002) + v7:CBool = IsBitEqual v5, v6 + CondBranch v7, bb3(v1, v3, v4), bb6() + bb6(): + Jump bb5(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v11:BasicObject = LoadArg :self@0 + v12:BasicObject = LoadArg :needle@1 + v13:NilClass = Const Value(nil) + Jump bb3(v11, v12, v13) + bb3(v20:BasicObject, v21:BasicObject, v22:BasicObject): + v25:Fixnum[0] = Const Value(0) + Jump bb5(v20, v21, v25) + bb4(): + EntryPoint JIT(1) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :needle@1 + v18:BasicObject = LoadArg :offset@2 + Jump bb5(v16, v17, v18) + bb5(v28:BasicObject, v29:BasicObject, v30:BasicObject): + PatchPoint MethodRedefined(String@0x1008, byteindex@0x1010, cme:0x1018) + v44:CPtr = GetEP 0 + v45:RubyValue = LoadField v44, :VM_ENV_DATA_INDEX_ME_CREF@0x1040 + v46:CallableMethodEntry[VALUE(0x1048)] = GuardBitEquals v45, Value(VALUE(0x1048)) + v47:RubyValue = LoadField v44, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 + v48:FalseClass = GuardBitEquals v47, Value(false) + v49:BasicObject = CCallVariadic v28, :String#byteindex@0x1058, v29, v30 + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_invokesuper_with_blockarg_remains_invokesuper() { + eval(" + class A + def foo + block_given? ? yield : 'no block' + end + end + + class B < A + def foo(&blk) + other_block = proc { 'different block' } + super(&other_block) + end + end + + B.new.foo { 'passed' }; B.new.foo { 'passed' } + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for explicit blockarg:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :blk@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:HeapBasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :blk@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:HeapBasicObject, v12:BasicObject, v13:NilClass): + PatchPoint NoSingletonClass(B@0x1008) + PatchPoint MethodRedefined(B@0x1008, proc@0x1010, cme:0x1018) + v39:ObjectSubclass[class_exact:B] = GuardType v11, ObjectSubclass[class_exact:B] recompile + v40:BasicObject = CCallWithFrame v39, :Kernel#proc@0x1040, block=0x1048 + v19:CPtr = GetEP 0 + v20:BasicObject = LoadField v19, :blk@0x1050 + SetLocal :other_block, l0, EP@3, v40 + v27:CPtr = GetEP 0 + v28:BasicObject = LoadField v27, :other_block@0x1051 + v30:BasicObject = InvokeSuper v39, 0x1058, v28 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_invokesuper_with_symbol_to_proc_remains_invokesuper() { + eval(" + class A + def foo(items, &blk) + items.map(&blk) + end + end + + class B < A + def foo(items) + super(items, &:succ) + end + end + + B.new.foo([1, 2, 3]); B.new.foo([1, 2, 3]) + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + assert!(!hir.contains("SendDirect"), "Should not optimize to SendDirect for symbol-to-proc:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :items@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:HeapBasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :items@1 + Jump bb3(v6, v7) + bb3(v9:HeapBasicObject, v10:BasicObject): + v16:StaticSymbol[:succ] = Const Value(VALUE(0x1008)) + v18:BasicObject = InvokeSuper v9, 0x1010, v10, v16 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_invokesuper_with_keyword_args_remains_invokesuper() { + eval(" + class A + def foo(attributes = {}) + @attributes = attributes + end + end + + class B < A + def foo(content = '') + super(content: content) + end + end + + B.new.foo('image data'); B.new.foo('image data') + "); + + let hir = hir_string_proc("B.new.method(:foo)"); + assert!(hir.contains("InvokeSuper "), "Expected unoptimized InvokeSuper but got:\n{hir}"); + + assert_snapshot!(hir, @" + fn foo@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :content@0x1000 + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v6:CBool = IsBitEqual v4, v5 + CondBranch v6, bb3(v1, v3), bb6() + bb6(): + Jump bb5(v1, v3) + bb2(): + EntryPoint JIT(0) + v10:HeapBasicObject = LoadArg :self@0 + v11:NilClass = Const Value(nil) + Jump bb3(v10, v11) + bb3(v17:HeapBasicObject, v18:BasicObject): + v21:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v22:StringExact = StringCopy v21 + Jump bb5(v17, v22) + bb4(): + EntryPoint JIT(1) + v14:HeapBasicObject = LoadArg :self@0 + v15:BasicObject = LoadArg :content@1 + Jump bb5(v14, v15) + bb5(v25:HeapBasicObject, v26:BasicObject): + v32:BasicObject = InvokeSuper v25, 0x1010, v26 # SendFallbackReason: super: complex argument passing to `super` call + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_infer_truthiness_from_branch() { + eval(" + def test(x) + if x + if x + if x + 3 + else + 4 + end + else + 5 + end + else + 6 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb7(), bb6(v9, v17) + bb7(): + v19:Truthy = RefineType v10, Truthy + CheckInterrupts + v38:Fixnum[3] = Const Value(3) + Return v38 + bb6(v43:BasicObject, v44:Falsy): + v48:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v48 + "); + } + + #[test] + fn specialize_polymorphic_send_iseq() { + set_call_threshold(4); + eval(" + class C + def foo = 3 + end + + class D + def foo = 4 + end + + def test o + o.foo + 2 + end + + test C.new; test D.new; test C.new; test D.new + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:11: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v55:Fixnum[3] = Const Value(3) + Jump bb4(v16, v17, v55) + bb6(): + v24:CBool = HasType v10, ObjectSubclass[class_exact:D] + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + PatchPoint NoSingletonClass(D@0x1040) + PatchPoint MethodRedefined(D@0x1040, foo@0x1010, cme:0x1048) + v56:Fixnum[4] = Const Value(4) + Jump bb4(v25, v26, v56) + bb8(): + v33:BasicObject = Send v10, :foo # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + v40:Fixnum[2] = Const Value(2) + PatchPoint MethodRedefined(Integer@0x1070, +@0x1078, cme:0x1080) + v59:Fixnum = GuardType v37, Fixnum recompile + v60:Fixnum = FixnumAdd v59, v40 + CheckInterrupts + Return v60 + "); + } + + #[test] + fn specialize_polymorphic_send_with_immediate() { + set_call_threshold(4); + eval(" + class C; end + + def test o + o.itself + end + + test C.new; test 3; test C.new; test 4 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:ObjectSubclass[class_exact:C] = RefineType v18, ObjectSubclass[class_exact:C] + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, itself@0x1010, cme:0x1018) + Jump bb4(v16, v17, v20) + bb6(): + v24:CBool = HasType v10, Fixnum + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:Fixnum = RefineType v27, Fixnum + PatchPoint MethodRedefined(Integer@0x1040, itself@0x1010, cme:0x1018) + Jump bb4(v25, v26, v29) + bb8(): + v33:BasicObject = Send v10, :itself # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_fixnum_and_bignum() { + // Fixnum and Bignum both have class Integer, but they should be + // treated as different types for polymorphic dispatch because + // Fixnum is an immediate and Bignum is a heap object. + set_call_threshold(4); + eval(" + def test x + x.to_s + end + + fixnum = 1 + bignum = 10**100 + test(fixnum) + test(bignum) + test(fixnum) + test(bignum) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, Fixnum + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:Fixnum = RefineType v18, Fixnum + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v46:StringExact = CCallVariadic v20, :Integer#to_s@0x1040 + Jump bb4(v16, v17, v46) + bb6(): + v24:CBool = HasType v10, Bignum + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:Bignum = RefineType v27, Bignum + PatchPoint MethodRedefined(Integer@0x1008, to_s@0x1010, cme:0x1018) + v49:StringExact = CCallVariadic v29, :Integer#to_s@0x1040 + Jump bb4(v25, v26, v49) + bb8(): + v33:BasicObject = Send v10, :to_s # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_flonum_and_heap_float() { + set_call_threshold(4); + eval(" + def test x + x.to_s + end + + flonum = 1.5 + heap_float = 1.7976931348623157e+308 + test(flonum) + test(heap_float) + test(flonum) + test(heap_float) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, Flonum + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:Flonum = RefineType v18, Flonum + PatchPoint MethodRedefined(Float@0x1008, to_s@0x1010, cme:0x1018) + v46:BasicObject = CCallWithFrame v20, :Float#to_s@0x1040 + Jump bb4(v16, v17, v46) + bb6(): + v24:CBool = HasType v10, HeapFloat + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:HeapFloat = RefineType v27, HeapFloat + PatchPoint MethodRedefined(Float@0x1008, to_s@0x1010, cme:0x1018) + v49:BasicObject = CCallWithFrame v29, :Float#to_s@0x1040 + Jump bb4(v25, v26, v49) + bb8(): + v33:BasicObject = Send v10, :to_s # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_static_and_dynamic_symbol() { + set_call_threshold(4); + eval(" + def test x + x.to_s + end + + static_sym = :foo + dynamic_sym = (\"zjit_dynamic_\" + Object.new.object_id.to_s).to_sym + test static_sym + test dynamic_sym + test static_sym + test dynamic_sym + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, StaticSymbol + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + v20:StaticSymbol = RefineType v18, StaticSymbol + PatchPoint MethodRedefined(Symbol@0x1008, to_s@0x1010, cme:0x1018) + v48:StringExact = InvokeBuiltin leaf <inline_expr>, v20 + Jump bb4(v16, v17, v48) + bb6(): + v24:CBool = HasType v10, DynamicSymbol + CondBranch v24, bb7(v9, v10, v10), bb8() + bb7(v25:BasicObject, v26:BasicObject, v27:BasicObject): + v29:DynamicSymbol = RefineType v27, DynamicSymbol + PatchPoint MethodRedefined(Symbol@0x1008, to_s@0x1010, cme:0x1018) + v49:StringExact = InvokeBuiltin leaf <inline_expr>, v29 + Jump bb4(v25, v26, v49) + bb8(): + v33:BasicObject = Send v10, :to_s # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v33) + bb4(v35:BasicObject, v36:BasicObject, v37:BasicObject): + CheckInterrupts + Return v37 + "); + } + + #[test] + fn specialize_polymorphic_send_iseq_duplicate_class_profiles() { + set_call_threshold(4); + eval(" + class C + def foo = 3 + end + + O1 = C.new + O1.instance_variable_set(:@foo, 1) + O2 = C.new + O2.instance_variable_set(:@bar, 2) + + def test o + o.foo + end + + test O1; test O2; test O1; test O2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CBool = HasType v10, ObjectSubclass[class_exact:C] + CondBranch v15, bb5(v9, v10, v10), bb6() + bb5(v16:BasicObject, v17:BasicObject, v18:BasicObject): + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) + v38:Fixnum[3] = Const Value(3) + Jump bb4(v16, v17, v38) + bb6(): + v24:BasicObject = Send v10, :foo # SendFallbackReason: SendWithoutBlock: polymorphic fallback + Jump bb4(v9, v10, v24) + bb4(v26:BasicObject, v27:BasicObject, v28:BasicObject): + CheckInterrupts + Return v28 + "); + } + + #[test] + fn upgrade_self_type_to_heap_after_setivar() { + // Snapshot the overflow path only when this build naturally keeps five + // ivars embedded and overflows on the next write. + let obj = eval(r#" + klass = Class.new do + def initialize + @v0 = 0 + @v1 = 1 + @v2 = 2 + @v3 = 3 + @v4 = 4 + end + + def test + @overflow = 1 + @after = 2 + end + end + + TEST = klass.instance_method(:test) + OBJ = klass.new + OBJ + "#); + // Skip builds where five ivars already force heap-backed storage. + if !obj.embedded_p() { + return; + } + + // Make sure the next write is the one that overflows into heap-backed + // storage, so this snapshot still exercises the self-type upgrade path. + let probe = eval(r#" + probe = OBJ.class.new + probe.instance_variable_set(:@overflow, 1) + probe + "#); + if probe.embedded_p() { + return; + } + eval("OBJ.test"); + assert_snapshot!(hir_string_proc("TEST"), @" + fn test@<compiled>:12: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@overflow, v10 + v14:HeapBasicObject = RefineType v6, HeapBasicObject + v17:Fixnum[2] = Const Value(2) + PatchPoint SingleRactorMode + v29:CShape = LoadField v14, :shape_id@0x1000 + v30:CShape[0x1001] = GuardBitEquals v29, CShape(0x1001) + v31:CPtr = LoadField v14, :as_heap@0x1002 + StoreField v31, :@after@0x1003, v17 + WriteBarrier v14, v17 + v34:CShape[0x1004] = Const CShape(0x1004) + StoreField v14, :shape_id@0x1000, v34 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn recompile_after_ep_escape_uses_ep_locals() { + // When a method creates a lambda, EP escapes to the heap. After + // invalidation and recompilation, the compiler must use EP-based + // locals (SetLocal/GetLocal) instead of SSA locals, because the + // spill target (stack) and the read target (heap EP) diverge. + eval(" + CONST = {}.freeze + def test_ep_escape(list, sep=nil, iter_method=:each) + sep ||= lambda { } + kwsplat = CONST + list.__send__(iter_method) {|*v| yield(*v) } + end + + test_ep_escape({a: 1}, nil, :each_pair) { |k, v| + test_ep_escape([1], lambda { }) { |x| } + } + test_ep_escape({a: 1}, nil, :each_pair) { |k, v| + test_ep_escape([1], lambda { }) { |x| } + } + "); + assert_snapshot!(hir_string("test_ep_escape"), @" + fn test_ep_escape@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :list@0x1000 + v4:BasicObject = LoadField v2, :sep@0x1001 + v5:BasicObject = LoadField v2, :iter_method@0x1002 + v6:NilClass = Const Value(nil) + v7:CPtr = LoadPC + v8:CPtr[CPtr(0x1003)] = Const CPtr(0x1003) + v9:CBool = IsBitEqual v7, v8 + CondBranch v9, bb3(v1, v3, v4, v5, v6), bb9() + bb9(): + v11:CPtr[CPtr(0x1004)] = Const CPtr(0x1004) + v12:CBool = IsBitEqual v7, v11 + CondBranch v12, bb5(v1, v3, v4, v5, v6), bb10() + bb10(): + Jump bb7(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :list@1 + v18:NilClass = Const Value(nil) + v19:NilClass = Const Value(nil) + v20:NilClass = Const Value(nil) + Jump bb3(v16, v17, v18, v19, v20) + bb3(v36:BasicObject, v37:BasicObject, v38:BasicObject, v39:BasicObject, v40:NilClass): + v43:NilClass = Const Value(nil) + SetLocal :sep, l0, EP@5, v43 + Jump bb5(v36, v37, v43, v39, v40) + bb4(): + EntryPoint JIT(1) + v23:BasicObject = LoadArg :self@0 + v24:BasicObject = LoadArg :list@1 + v25:BasicObject = LoadArg :sep@2 + v26:NilClass = Const Value(nil) + v27:NilClass = Const Value(nil) + Jump bb5(v23, v24, v25, v26, v27) + bb5(v47:BasicObject, v48:BasicObject, v49:BasicObject, v50:BasicObject, v51:NilClass): + v54:StaticSymbol[:each] = Const Value(VALUE(0x1008)) + SetLocal :iter_method, l0, EP@4, v54 + Jump bb7(v47, v48, v49, v54, v51) + bb6(): + EntryPoint JIT(2) + v30:BasicObject = LoadArg :self@0 + v31:BasicObject = LoadArg :list@1 + v32:BasicObject = LoadArg :sep@2 + v33:BasicObject = LoadArg :iter_method@3 + v34:NilClass = Const Value(nil) + Jump bb7(v30, v31, v32, v33, v34) + bb7(v58:BasicObject, v59:BasicObject, v60:BasicObject, v61:BasicObject, v62:NilClass): + CheckInterrupts + v68:CBool = Test v60 + v69:Truthy = RefineType v60, Truthy + CondBranch v68, bb8(v58, v59, v69, v61, v62), bb11() + bb11(): + v71:Falsy = RefineType v60, Falsy + PatchPoint MethodRedefined(Object@0x1010, lambda@0x1018, cme:0x1020) + v118:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v58, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile + v119:BasicObject = CCallWithFrame v118, :Kernel#lambda@0x1048, block=0x1050 + v75:CPtr = GetEP 0 + v76:BasicObject = LoadField v75, :list@0x1001 + v78:BasicObject = LoadField v75, :iter_method@0x1058 + v79:BasicObject = LoadField v75, :kwsplat@0x1059 + SetLocal :sep, l0, EP@5, v119 + Jump bb8(v118, v76, v119, v78, v79) + bb8(v83:BasicObject, v84:BasicObject, v85:BasicObject, v86:BasicObject, v87:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1060, CONST) + v115:HashExact[VALUE(0x1068)] = Const Value(VALUE(0x1068)) + SetLocal :kwsplat, l0, EP@3, v115 + v96:CPtr = GetEP 0 + v97:BasicObject = LoadField v96, :list@0x1001 + v99:CPtr = GetEP 0 + v100:BasicObject = LoadField v99, :iter_method@0x1058 + v102:BasicObject = Send v97, 0x1070, :__send__, v100 # SendFallbackReason: Send: unsupported method type Optimized + v103:CPtr = GetEP 0 + v104:BasicObject = LoadField v103, :list@0x1001 + v105:BasicObject = LoadField v103, :sep@0x1002 + v106:BasicObject = LoadField v103, :iter_method@0x1058 + v107:BasicObject = LoadField v103, :kwsplat@0x1059 + CheckInterrupts + Return v102 + "); + } + + #[test] + fn test_array_each() { + eval("[1, 2, 3].each { |x| x }"); + assert_snapshot!(hir_string_proc("Array.instance_method(:each)"), @" + fn each@<internal:array>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + v15:TrueClass|NilClass = Defined yield, v13 + v17:CBool = Test v15 + CondBranch v17, bb9(), bb4(v8, v9) + bb9(): + v35:Fixnum[0] = Const Value(0) + Jump bb8(v8, v35) + bb8(v48:BasicObject, v49:Fixnum): + v84:Array = RefineType v48, Array + v85:CInt64 = ArrayLength v84 + v86:Fixnum = BoxFixnum v85 + v87:BoolExact = FixnumGe v49, v86 + v54:CBool = Test v87 + CondBranch v54, bb10(), bb7(v48, v49) + bb10(): + CheckInterrupts + Return v48 + bb7(v67:BasicObject, v68:Fixnum): + v88:Array = RefineType v67, Array + v89:CInt64 = UnboxFixnum v68 + v90:BasicObject = ArrayAref v88, v89 + v74:BasicObject = InvokeBlock, v90 # SendFallbackReason: InvokeBlock: not yet specialized + v91:Fixnum[1] = Const Value(1) + v92:Fixnum = FixnumAdd v68, v91 + PatchPoint NoEPEscape(each) + Jump bb8(v67, v92) + bb4(v23:BasicObject, v24:NilClass): + v28:BasicObject = InvokeBuiltin <inline_expr>, v23 + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_delete_duplicate_store() { + eval(" + class C + def initialize + a = 1 + @a = a + @a = a + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v35:HeapBasicObject = GuardType v8, HeapBasicObject + v36:CShape = LoadField v35, :shape_id@0x1000 + v37:CShape[0x1001] = GuardBitEquals v36, CShape(0x1001) recompile + StoreField v35, :@a@0x1002, v13 + WriteBarrier v35, v13 + v40:CShape[0x1003] = Const CShape(0x1003) + StoreField v35, :shape_id@0x1000, v40 + PatchPoint NoEPEscape(initialize) + PatchPoint SingleRactorMode + WriteBarrier v35, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_remove_duplicate_store_with_non_effectful_insns_between() { + eval(" + class C + def initialize + a = 1 + @a = a + b = 5 + b += a + @a = a + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb3(v6, v7, v8) + bb3(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v49:HeapBasicObject = GuardType v10, HeapBasicObject + v50:CShape = LoadField v49, :shape_id@0x1000 + v51:CShape[0x1001] = GuardBitEquals v50, CShape(0x1001) recompile + StoreField v49, :@a@0x1002, v16 + WriteBarrier v49, v16 + v54:CShape[0x1003] = Const CShape(0x1003) + StoreField v49, :shape_id@0x1000, v54 + v26:Fixnum[5] = Const Value(5) + PatchPoint NoEPEscape(initialize) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v65:Fixnum[6] = Const Value(6) + PatchPoint SingleRactorMode + WriteBarrier v49, v16 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_remove_two_stores() { + eval(" + class C + def initialize + a = 1 + @a = a + @a = a + @a = a + end + end + + C.new + "); + assert_snapshot!(hir_string_proc("C.instance_method(:initialize)"), @" + fn initialize@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + v43:HeapBasicObject = GuardType v8, HeapBasicObject + v44:CShape = LoadField v43, :shape_id@0x1000 + v45:CShape[0x1001] = GuardBitEquals v44, CShape(0x1001) recompile + StoreField v43, :@a@0x1002, v13 + WriteBarrier v43, v13 + v48:CShape[0x1003] = Const CShape(0x1003) + StoreField v43, :shape_id@0x1000, v48 + PatchPoint NoEPEscape(initialize) + PatchPoint SingleRactorMode + WriteBarrier v43, v13 + WriteBarrier v43, v13 + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_exit_from_function_stub_for_opt_keyword_callee() { + // We have a SendDirect to a callee that fails to compile, + // so the function stub has to take care of exiting to + // interpreter. + eval(" + def target(a = binding.local_variable_get(:a), b: nil) + ::RubyVM::ZJIT.induce_compile_failure! + [a, b] + end + + def entry = target(b: -1) + + raise 'wrong' unless [nil, -1] == entry + raise 'wrong' unless [nil, -1] == entry + "); + + crate::hir::tests::hir_build_tests::assert_compile_fails("target", ParseError::DirectiveInduced); + let hir = hir_string("entry"); + assert!(hir.contains("SendDirect"), "{hir}"); + } + + #[test] + fn test_exit_from_function_stub_for_lead_opt() { + // We have a SendDirect to a callee that fails to compile, + // so the function stub has to take care of exiting to + // interpreter. + let result = eval(" + def target(_required, a = a, b = b) + ::RubyVM::ZJIT.induce_compile_failure! + a + end + + def entry = target(1) + + entry + entry + "); + assert_eq!(Qnil, result); + + crate::hir::tests::hir_build_tests::assert_compile_fails("target", ParseError::DirectiveInduced); + let hir = hir_string("entry"); + assert!(hir.contains("SendDirect"), "{hir}"); + } + + #[test] + fn test_recompile_no_profile_send() { + // Test the SideExit -> recompile flow: a no-profile send becomes a SideExit, + // the exit profiles the send, triggers recompilation, and the new version + // optimizes it to SendDirect. + eval(" + def greet_recompile(x) = x.to_s + def test_no_profile_recompile(flag) + if flag + greet_recompile(42) + else + 'hello' + end + end + "); + + // With call_threshold=2, num_profiles=1: + // 1st call profiles (flag=false, so greet is never reached) + // 2nd call compiles (greet has no profile data -> SideExit recompile) + eval("test_no_profile_recompile(false); test_no_profile_recompile(false)"); + + // Now call with flag=true. This hits the SideExit, which profiles + // the send and invalidates the ISEQ for recompilation. + eval("test_no_profile_recompile(true)"); + + // After profiling via the side exit, rebuilding HIR should now + // have a SendDirect for greet_recompile instead of SideExit. + assert_snapshot!(hir_string("test_no_profile_recompile"), @r" + fn test_no_profile_recompile@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :flag@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :flag@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb5(), bb4(v9, v17) + bb5(): + v19:Truthy = RefineType v10, Truthy + v23:Fixnum[42] = Const Value(42) + PatchPoint MethodRedefined(Object@0x1008, greet_recompile@0x1010, cme:0x1018) + v43:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v44:BasicObject = SendDirect v43, 0x1040, :greet_recompile (0x1050), v23 + CheckInterrupts + Return v44 + bb4(v30:BasicObject, v31:Falsy): + v35:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) + v36:StringExact = StringCopy v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_no_profile_send_on_final_version() { + // On the final ISEQ version (MAX_ISEQ_VERSIONS reached), no-profile sends should + // remain as Send fallbacks instead of being converted to SideExits, since recompilation + // is no longer possible and SideExits would fire every time without benefit. + // + // Use call_threshold=3 to ensure the method is auto-compiled before hir_string() builds + // the HIR. The auto-compile creates version 1, and hir_string() creates version 2 + // (= MAX_ISEQ_VERSIONS), so this is the final version. + set_call_threshold(3); + eval(" + def greet_final(x) = x.to_s + def test_final_version(flag) + if flag + greet_final(42) + else + 'hello' + end + end + "); + // Call enough times to trigger auto-compilation. flag=false so greet_final is never + // reached and has no profile data. + eval("3.times { test_final_version(false) }"); + + // On the final version, greet_final should be a Send fallback, not a SideExit. + assert_snapshot!(hir_string("test_final_version"), @r" + fn test_final_version@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :flag@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :flag@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb5(), bb4(v9, v17) + bb5(): + v19:Truthy = RefineType v10, Truthy + v23:Fixnum[42] = Const Value(42) + v25:BasicObject = Send v9, :greet_final, v23 # SendFallbackReason: SendWithoutBlock: no profile data available + CheckInterrupts + Return v25 + bb4(v30:BasicObject, v31:Falsy): + v35:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v36:StringExact = StringCopy v35 + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_invokeblock_ifunc() { + eval(" + class IFuncTestList + include Enumerable + def each + yield 1 + yield 2 + end + end + IFuncTestList.new.map { |x| x } + "); + assert_snapshot!(hir_string_proc("IFuncTestList.instance_method(:each)"), @" + fn each@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:CPtr = GetEP 0 + v13:CInt64 = LoadField v12, :VM_ENV_DATA_INDEX_SPECVAL@0x1000 + v14:CInt64[3] = Const CInt64(3) + v15:CInt64 = IntAnd v13, v14 + v16:CInt64[3] = Const CInt64(3) + v17:CBool = IsBitEqual v15, v16 + CondBranch v17, bb5(), bb6() + bb5(): + v20:BasicObject = InvokeBlockIfunc v13, v10 + Jump bb4(v20) + bb6(): + v22:BasicObject = InvokeBlock, v10 # SendFallbackReason: InvokeBlock: not yet specialized + Jump bb4(v22) + bb4(v18:BasicObject): + v27:Fixnum[2] = Const Value(2) + v29:CPtr = GetEP 0 + v30:CInt64 = LoadField v29, :VM_ENV_DATA_INDEX_SPECVAL@0x1000 + v31:CInt64[3] = Const CInt64(3) + v32:CInt64 = IntAnd v30, v31 + v33:CInt64[3] = Const CInt64(3) + v34:CBool = IsBitEqual v32, v33 + CondBranch v34, bb8(), bb9() + bb8(): + v37:BasicObject = InvokeBlockIfunc v30, v27 + Jump bb7(v37) + bb9(): + v39:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized + Jump bb7(v39) + bb7(v35:BasicObject): + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_dedup_guard_type() { + // Two subtractions on the same Fixnum operand `n` each require a + // GuardType n, Fixnum. The second guard is redundant and should be + // eliminated by fold_constants. + eval(" + def test(n) + (n - 1) + (n - 2) + end + test 1; test 2 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :n@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :n@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, -@0x1010, cme:0x1018) + v35:Fixnum = GuardType v10, Fixnum recompile + v36:Fixnum = FixnumSub v35, v15 + v21:Fixnum[2] = Const Value(2) + v40:Fixnum = FixnumSub v35, v21 + PatchPoint MethodRedefined(Integer@0x1008, +@0x1040, cme:0x1048) + v44:Fixnum = FixnumAdd v36, v40 + CheckInterrupts + Return v44 + "); + } + + #[test] + fn test_dedup_guard_type_across_cfg_join() { + eval(" + def test(n, cond) + if cond + a = n + 1 + else + a = n + 2 + end + n + a + end + test(1, true); test(1, false) + "); + let hir = hir_string("test"); + let guard_count = hir.matches("GuardType").count(); + assert_eq!( + guard_count, 2, + "expected 2 GuardType instructions after cross-block dedup, found {guard_count}\n\nHIR:\n{hir}" + ); + } + + #[test] + fn test_forward_guard_through_conditional_branch() { + eval(" + def test(n, a, b) + if a + if b + n + 1 + else + n + 2 + end + else + n + 3 + end + end + test(1, true, true); test(1, true, false); test(1, false, false) + "); + let hir = hir_string("test"); + let guard_count = hir.matches("GuardType").count(); + assert!( + guard_count <= 3, + "expected at most 3 GuardType instructions (one per leaf branch) after forwarding through conditional branches, found {guard_count}\n\nHIR:\n{hir}" + ); + } + + #[test] + fn test_no_forward_when_no_guard_in_branches() { + let src = " + def test(n, cond) + a = if cond then 1 else 2 end + n + a + end + test(1, true); test(1, false) + "; + eval(src); + let hir = hir_string("test"); + let guard_count = hir.matches("GuardType").count(); + assert_eq!( + guard_count, 1, + "expected 1 GuardType (merge block only), found {guard_count}\n\nHIR:\n{hir}" + ); + } + + #[test] + fn test_infer_types_across_non_maximal_basic_blocks() { + // Previous worklist-based type inference only worked for maximal SSA. This is a regression + // test for hanging. + eval(" + class TheClass + def set_value_loop + i = 0 + while i < 10 + @levar ||= i + i += 1 + end + end + end + 3.times do |i| + TheClass.new.set_value_loop + end + "); + assert_snapshot!(hir_string_proc("TheClass.instance_method(:set_value_loop)"), @" + fn set_value_loop@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:HeapBasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:HeapBasicObject, v9:NilClass): + v13:Fixnum[0] = Const Value(0) + CheckInterrupts + Jump bb6(v8, v13) + bb6(v19:HeapBasicObject, v20:Fixnum): + v24:Fixnum[10] = Const Value(10) + PatchPoint MethodRedefined(Integer@0x1000, <@0x1008, cme:0x1010) + v110:BoolExact = FixnumLt v20, v24 + CheckInterrupts + v30:CBool = Test v110 + CondBranch v30, bb4(v19, v20), bb7() + bb4(v40:HeapBasicObject, v41:Fixnum): + PatchPoint SingleRactorMode + v47:CUInt64 = LoadField v40, :RBASIC_FLAGS@0x1038 + v49:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v50:CPtr[CPtr(0x1039)] = Const CPtr(0x1039) + v51 = RefineType v50, CUInt64 + v52:CInt64 = IntAnd v47, v49 + v53:CBool = IsBitEqual v52, v51 + CondBranch v53, bb9(), bb10() + bb9(): + v55:BasicObject = LoadField v40, :@levar@0x103a + Jump bb8(v55) + bb10(): + v57:CUInt64[0xffffffff0000001f] = Const CUInt64(0xffffffff0000001f) + v58:CPtr[CPtr(0x103b)] = Const CPtr(0x103b) + v59 = RefineType v58, CUInt64 + v60:CInt64 = IntAnd v47, v57 + v61:CBool = IsBitEqual v60, v59 + CondBranch v61, bb11(), bb12() + bb11(): + v63:NilClass = Const Value(nil) + Jump bb8(v63) + bb12(): + v97:CShape = LoadField v40, :shape_id@0x103c + v98:CShape[0x103d] = GuardBitEquals v97, CShape(0x103d) recompile + v99:BasicObject = LoadField v40, :@levar@0x103a + Jump bb8(v99) + bb8(v48:BasicObject): + CheckInterrupts + v69:CBool = Test v48 + CondBranch v69, bb5(v40, v41), bb13() + bb13(): + PatchPoint NoEPEscape(set_value_loop) + PatchPoint SingleRactorMode + v101:CShape = LoadField v40, :shape_id@0x103c + v102:CShape[0x103e] = GuardBitEquals v101, CShape(0x103e) recompile + StoreField v40, :@levar@0x103a, v41 + WriteBarrier v40, v41 + v105:CShape[0x103d] = Const CShape(0x103d) + StoreField v40, :shape_id@0x103c, v105 + Jump bb5(v40, v41) + bb5(v81:HeapBasicObject, v82:Fixnum): + PatchPoint NoEPEscape(set_value_loop) + v89:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1000, +@0x103f, cme:0x1040) + v114:Fixnum = FixnumAdd v82, v89 + Jump bb6(v81, v114) + bb7(): + v35:NilClass = Const Value(nil) + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_float_nan_p_annotation() { + eval(r#" + def test(x) = x.nan? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, nan?@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum recompile + v24:BoolExact = CCall v23, :Float#nan?@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_finite_p_annotation() { + eval(r#" + def test(x) = x.finite? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, finite?@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum recompile + v24:BoolExact = CCall v23, :Float#finite?@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_infinite_p_annotation() { + eval(r#" + def test(x) = x.infinite? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, infinite?@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum recompile + v24:NilClass|Fixnum = CCall v23, :Float#infinite?@0x1040 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_integer_even_p_annotation() { + eval(r#" + def test(x) = x.even? + test(2) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, even?@0x1010, cme:0x1018) + v22:Fixnum = GuardType v10, Fixnum recompile + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_integer_odd_p_annotation() { + eval(r#" + def test(x) = x.odd? + test(3) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Integer@0x1008, odd?@0x1010, cme:0x1018) + v22:Fixnum = GuardType v10, Fixnum recompile + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_zero_p_annotation() { + eval(r#" + def test(x) = x.zero? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, zero?@0x1010, cme:0x1018) + v22:Flonum = GuardType v10, Flonum recompile + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_positive_p_annotation() { + eval(r#" + def test(x) = x.positive? + test(1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, positive?@0x1010, cme:0x1018) + v22:Flonum = GuardType v10, Flonum recompile + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_negative_p_annotation() { + eval(r#" + def test(x) = x.negative? + test(-1.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, negative?@0x1010, cme:0x1018) + v22:Flonum = GuardType v10, Flonum recompile + v24:BoolExact = InvokeBuiltin leaf <inline_expr>, v22 + CheckInterrupts + Return v24 + "); + } + #[test] + fn test_float_add_inline() { + eval(r#" + def test(a, b) = a + b + test(1.0, 2.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, +@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum recompile + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatAdd v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_mul_inline() { + eval(r#" + def test(a, b) = a * b + test(1.5, 2.5) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, *@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum recompile + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatMul v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_sub_inline() { + eval(r#" + def test(a, b) = a - b + test(5.0, 3.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, -@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum recompile + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatSub v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_div_inline() { + eval(r#" + def test(a, b) = a / b + test(10.0, 3.0) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, /@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum recompile + v29:Flonum = GuardType v13, Flonum + v30:Float = FloatDiv v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_float_to_i_inline() { + eval(r#" + def test(a) = a.to_i + test(3.7) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, to_i@0x1010, cme:0x1018) + v23:Flonum = GuardType v10, Flonum recompile + v24:Integer = FloatToInt v23 + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_float_mul_fixnum_inline() { + eval(r#" + def test(a, b) = a * b + test(1.5, 3) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Float@0x1008, *@0x1010, cme:0x1018) + v28:Flonum = GuardType v12, Flonum recompile + v29:Fixnum = GuardType v13, Fixnum + v30:Float = FloatMul v28, v29 + CheckInterrupts + Return v30 + "); + } + + #[test] + fn test_elide_repeated_heap_object_guards() { + eval(r#" + C = Struct.new(:var) + def test(obj) + sum = 0 + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum += obj.var + sum + end + test(C.new(3)) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :obj@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:Fixnum[0] = Const Value(0) + PatchPoint NoSingletonClass(C@0x1008) + PatchPoint MethodRedefined(C@0x1008, var@0x1010, cme:0x1018) + v138:ObjectSubclass[class_exact:C] = GuardType v12, ObjectSubclass[class_exact:C] recompile + v139:BasicObject = LoadField v138, :var@0x1040 + PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) + v179:Fixnum = GuardType v139, Fixnum + PatchPoint NoEPEscape(test) + v185:Fixnum = FixnumAdd v179, v179 + v190:Fixnum = FixnumAdd v185, v179 + v195:Fixnum = FixnumAdd v190, v179 + v200:Fixnum = FixnumAdd v195, v179 + v205:Fixnum = FixnumAdd v200, v179 + v210:Fixnum = FixnumAdd v205, v179 + v215:Fixnum = FixnumAdd v210, v179 + v220:Fixnum = FixnumAdd v215, v179 + v225:Fixnum = FixnumAdd v220, v179 + CheckInterrupts + Return v225 + "); + } + + #[test] + fn test_dont_fold_array_length() { + eval(r#" + A = [1, 2, 3, 4] + def test = A.length + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v21:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, length@0x1018, cme:0x1020) + v26:CInt64 = ArrayLength v21 + v27:Fixnum = BoxFixnum v26 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_fold_frozen_array_length() { + eval(r#" + A = [1, 2, 3, 4].freeze + def test = A.length + test + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + PatchPoint StableConstantNames(0x1000, A) + v21:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + PatchPoint NoSingletonClass(Array@0x1010) + PatchPoint MethodRedefined(Array@0x1010, length@0x1018, cme:0x1020) + v28:CInt64[4] = Const CInt64(4) + v27:Fixnum = BoxFixnum v28 + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_elide_test_of_box_bool() { + eval(r#" + def test(a, b) + if a == b + 3 + else + 4 + end + end + test(:a, :b) + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint MethodRedefined(Symbol@0x1008, ==@0x1010, cme:0x1018) + v48:StaticSymbol = GuardType v12, StaticSymbol recompile + v49:CBool = IsBitEqual v48, v13 + v50:BoolExact = BoxBool v49 + CheckInterrupts + CondBranch v49, bb5(), bb4(v11, v48, v13) + bb5(): + v29:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v29 + bb4(v34:BasicObject, v35:StaticSymbol, v36:BasicObject): + v40:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v40 + "); + } + + #[test] + fn test_trigger_guard_type_recompilation() { + eval(" + class C + def f(x) + @a = 1 + y = x + 1 + @a = y + end + end + + # As of 06/04/2026, zjit/src/options.rs uses 5 as the default number of profiles + # Let's pick a number that is reasonably larger to ensure compilation, even if + # the default value changes a bit + num_to_compile = 30 + + c = C.new + + # Repeatedly call an integer until this fast path gets JITed + num_to_compile.times { c.f(1) } + + "); + + let intermediate_hir = hir_string_proc("C.new.method(:f)"); + + eval(" + # Supposed to be the same as the earlier Ruby method in this test + num_to_compile = 30 + c = C.new + # Call this with a float in order to trigger a guard failure + # Do this enough times to cause a recompilation + num_to_compile.times { c.f(1.5) } + "); + + let final_hir = hir_string_proc("C.new.method(:f)"); + + assert_snapshot!(format!("{intermediate_hir}\n{final_hir}"), @" + fn f@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:HeapBasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:HeapBasicObject, v12:BasicObject, v13:NilClass): + v17:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v11, :@a, v17 + PatchPoint NoEPEscape(f) + v27:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1008, +@0x1010, cme:0x1018) + v46:Fixnum = GuardType v12, Fixnum recompile + v47:Fixnum = FixnumAdd v46, v27 + PatchPoint SingleRactorMode + SetIvar v11, :@a, v47 + CheckInterrupts + Return v47 + + fn f@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:HeapBasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:HeapBasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:HeapBasicObject, v12:BasicObject, v13:NilClass): + v17:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v11, :@a, v17 + PatchPoint NoEPEscape(f) + v27:Fixnum[1] = Const Value(1) + v30:CBool = HasType v12, Flonum + CondBranch v30, bb5(v11, v12, v13, v12, v27), bb6() + bb5(v31:HeapBasicObject, v32:BasicObject, v33:NilClass, v34:BasicObject, v35:Fixnum[1]): + v37:Flonum = RefineType v34, Flonum + PatchPoint MethodRedefined(Float@0x1008, +@0x1010, cme:0x1018) + v74:Float = FloatAdd v37, v35 + Jump bb4(v31, v32, v33, v74) + bb6(): + v41:CBool = HasType v12, Fixnum + CondBranch v41, bb7(v11, v12, v13, v12, v27), bb8() + bb7(v42:HeapBasicObject, v43:BasicObject, v44:NilClass, v45:BasicObject, v46:Fixnum[1]): + v48:Fixnum = RefineType v45, Fixnum + PatchPoint MethodRedefined(Integer@0x1040, +@0x1010, cme:0x1048) + v77:Fixnum = FixnumAdd v48, v46 + Jump bb4(v42, v43, v44, v77) + bb8(): + PatchPoint MethodRedefined(Float@0x1008, +@0x1010, cme:0x1018) + v80:Flonum = GuardType v12, Flonum recompile + v81:Float = FloatAdd v80, v27 + Jump bb4(v11, v80, v13, v81) + bb4(v54:HeapBasicObject, v55:BasicObject, v56:NilClass, v57:Float|Fixnum): + PatchPoint SingleRactorMode + SetIvar v54, :@a, v57 + CheckInterrupts + Return v57 + "); + } +} diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs new file mode 100644 index 0000000000..09ca3687e2 --- /dev/null +++ b/zjit/src/hir/tests.rs @@ -0,0 +1,6433 @@ +#[cfg(test)] +use super::*; + +#[cfg(test)] +mod snapshot_tests { + use super::*; + use insta::assert_snapshot; + + #[track_caller] + fn hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[track_caller] + fn optimized_hir_string(method: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(&format!("{}.method(:{})", "self", method))); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let mut function = iseq_to_hir(iseq).unwrap(); + function.optimize(); + function.validate().unwrap(); + format!("{}", FunctionPrinter::with_snapshot(&function)) + } + + #[test] + fn test_remove_redundant_patch_points() { + eval(" + def test = 1 + 2 + 3 + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:2: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v13:Any = Snapshot FrameState { pc: 0x1008, stack: [v10, v12], locals: [] } + PatchPoint MethodRedefined(Integer@0x1010, +@0x1018, cme:0x1020) + v35:Fixnum[6] = Const Value(6) + v21:Any = Snapshot FrameState { pc: 0x1048, stack: [v35], locals: [] } + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [a=v12, b=v13] } + v15:Any = Snapshot FrameState { pc: 0x1010, stack: [], locals: [a=v12, b=v13] } + PatchPoint NoTracePoint + v17:Any = Snapshot FrameState { pc: 0x1018, stack: [v12], locals: [a=v12, b=v13] } + v18:Any = Snapshot FrameState { pc: 0x1020, stack: [v12, v13], locals: [a=v12, b=v13] } + v19:ArrayExact = NewArray v12, v13 + v20:Any = Snapshot FrameState { pc: 0x1028, stack: [v19], locals: [a=v12, b=v13] } + PatchPoint NoTracePoint + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_send_direct_with_reordered_kwargs_has_snapshot() { + eval(" + def foo(a:, b:, c:) = [a, b, c] + def test = foo(c: 3, a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:3: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[3] = Const Value(3) + v13:Fixnum[1] = Const Value(1) + v15:Fixnum[2] = Const Value(2) + v16:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15], locals: [] } + v23:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v13, v15, v11], locals: [] } + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile + v26:BasicObject = SendDirect v25, 0x1048, :foo (0x1058), v13, v15, v11 + v18:Any = Snapshot FrameState { pc: 0x1060, stack: [v26], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_send_direct_with_kwargs_in_order_has_snapshot() { + eval(" + def foo(a:, b:) = [a, b] + def test = foo(a: 1, b: 2) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:3: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[1] = Const Value(1) + v13:Fixnum[2] = Const Value(2) + v14:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } + PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) + v22:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile + v23:BasicObject = SendDirect v22, 0x1048, :foo (0x1058), v11, v13 + v16:Any = Snapshot FrameState { pc: 0x1060, stack: [v23], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_send_direct_with_many_kwargs_no_reorder_snapshot() { + eval(" + def foo(five, six, a:, b:, c:, d:, e:, f:) = [a, b, c, d, five, six, e, f] + def test = foo(5, 6, d: 4, c: 3, a: 1, b: 2, e: 7, f: 8) + test + test + "); + assert_snapshot!(optimized_hir_string("test"), @" + fn test@<compiled>:3: + bb0(): + Entries bb1, bb2 + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v8:Any = Snapshot FrameState { pc: 0x1000, stack: [], locals: [] } + PatchPoint NoTracePoint + v11:Fixnum[5] = Const Value(5) + v13:Fixnum[6] = Const Value(6) + v15:Fixnum[4] = Const Value(4) + v17:Fixnum[3] = Const Value(3) + v19:Fixnum[1] = Const Value(1) + v21:Fixnum[2] = Const Value(2) + v23:Fixnum[7] = Const Value(7) + v25:Fixnum[8] = Const Value(8) + v26:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13, v15, v17, v19, v21, v23, v25], locals: [] } + v27:BasicObject = Send v6, :foo, v11, v13, v15, v17, v19, v21, v23, v25 # SendFallbackReason: Too many arguments for LIR + v28:Any = Snapshot FrameState { pc: 0x1010, stack: [v27], locals: [] } + PatchPoint NoTracePoint + CheckInterrupts + Return v27 + "); + } +} + +#[cfg(test)] +pub(crate) mod hir_build_tests { + use super::*; + use crate::options::set_call_threshold; + use insta::assert_snapshot; + + fn iseq_contains_opcode(iseq: IseqPtr, expected_opcode: u32) -> bool { + let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; + let mut insn_idx = 0; + while insn_idx < iseq_size { + // Get the current pc and opcode + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + + // try_into() call below is unfortunate. Maybe pick i32 instead of usize for opcodes. + let opcode: u32 = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } + .try_into() + .unwrap(); + if opcode == expected_opcode { + return true; + } + insn_idx += insn_len(opcode as usize); + } + false + } + + #[track_caller] + pub fn assert_contains_opcode(method: &str, opcode: u32) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + + #[track_caller] + fn assert_contains_opcodes(method: &str, opcodes: &[u32]) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + for &opcode in opcodes { + assert!(iseq_contains_opcode(iseq, opcode), "iseq {method} does not contain {}", insn_name(opcode as usize)); + } + } + + /// Combine multiple hir_string() results to match all of them at once, which allows + /// us to avoid running the set of zjit-test -> zjit-test-update multiple times. + #[macro_export] + macro_rules! hir_strings { + ($( $s:expr ),+ $(,)?) => {{ + vec![$( hir_string($s) ),+].join("\n") + }}; + } + + #[track_caller] + fn hir_string(method: &str) -> String { + hir_string_proc(&format!("{}.method(:{})", "self", method)) + } + + #[track_caller] + fn hir_string_proc(proc: &str) -> String { + let iseq = crate::cruby::with_rubyvm(|| get_proc_iseq(proc)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let function = iseq_to_hir(iseq).unwrap(); + hir_string_function(&function) + } + + #[track_caller] + fn hir_string_function(function: &Function) -> String { + format!("{}", FunctionPrinter::without_snapshot(function)) + } + + #[track_caller] + pub fn assert_compile_fails(method: &str, reason: ParseError) { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("self", method)); + unsafe { crate::cruby::rb_zjit_profile_disable(iseq) }; + let result = iseq_to_hir(iseq); + assert!(result.is_err(), "Expected an error but successfully compiled to HIR: {}", FunctionPrinter::without_snapshot(&result.unwrap())); + assert_eq!(result.unwrap_err(), reason); + } + + #[test] + fn test_compile_optional() { + eval("def test(x=1) = 123"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v6:CBool = IsBitEqual v4, v5 + CondBranch v6, bb3(v1, v3), bb6() + bb6(): + Jump bb5(v1, v3) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:NilClass = Const Value(nil) + Jump bb3(v10, v11) + bb3(v17:BasicObject, v18:BasicObject): + v21:Fixnum[1] = Const Value(1) + Jump bb5(v17, v21) + bb4(): + EntryPoint JIT(1) + v14:BasicObject = LoadArg :self@0 + v15:BasicObject = LoadArg :x@1 + Jump bb5(v14, v15) + bb5(v24:BasicObject, v25:BasicObject): + v29:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_putobject() { + eval("def test = 123"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[123] = Const Value(123) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_checkmatch_case() { + eval(r#" + def test(o) + case o + in Integer + 1 + else + 2 + end + end + test(1) + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + v18:BasicObject = GetConstantPath 0x1008 + v20:BasicObject = CheckMatch v10, v18, CASE + CheckInterrupts + v23:CBool = Test v20 + v24:Truthy = RefineType v20, Truthy + CondBranch v23, bb4(v9, v10, v14, v10), bb5() + bb4(v36:BasicObject, v37:BasicObject, v38:NilClass, v39:BasicObject): + v44:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v44 + bb5(): + v26:Falsy = RefineType v20, Falsy + v31:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_checkmatch_case_splat_array() { + eval(r#" + def test(o) + case o + when *[1, 2] + 1 + else + 2 + end + end + test(1) + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :o@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v17:ArrayExact = ArrayDup v16 + v19:BasicObject = CheckMatch v10, v17, CASE|ARRAY + CheckInterrupts + v22:CBool = Test v19 + v23:Truthy = RefineType v19, Truthy + CondBranch v22, bb4(v9, v10, v10), bb5() + bb4(v34:BasicObject, v35:BasicObject, v36:BasicObject): + v41:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v41 + bb5(): + v25:Falsy = RefineType v19, Falsy + v29:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_checkmatch_when_splat_array() { + eval(r#" + def test + case + when *[1, 2] + 1 + else + 2 + end + end + test + "#); + assert_contains_opcode("test", YARVINSN_checkmatch); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v13:ArrayExact = ArrayDup v12 + v15:BasicObject = CheckMatch v10, v13, WHEN|ARRAY + CheckInterrupts + v18:CBool = Test v15 + v19:Truthy = RefineType v15, Truthy + CondBranch v18, bb4(v6), bb5() + bb4(v29:BasicObject): + v33:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v33 + bb5(): + v21:Falsy = RefineType v15, Falsy + v24:Fixnum[2] = Const Value(2) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_new_array() { + eval("def test = []"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact = NewArray + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_array_with_element() { + eval("def test(a) = [a]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = NewArray v10 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_new_array_with_elements() { + eval("def test(a, b) = [a, b]"); + assert_contains_opcode("test", YARVINSN_newarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_new_range_inclusive_with_one_element() { + eval("def test(a) = (a..10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v17:RangeExact = NewRange v10 NewRangeInclusive v15 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_new_range_inclusive_with_two_elements() { + eval("def test(a, b) = (a..b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:RangeExact = NewRange v12 NewRangeInclusive v13 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_new_range_exclusive_with_one_element() { + eval("def test(a) = (a...10)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[10] = Const Value(10) + v17:RangeExact = NewRange v10 NewRangeExclusive v15 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_new_range_exclusive_with_two_elements() { + eval("def test(a, b) = (a...b)"); + assert_contains_opcode("test", YARVINSN_newrange); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:RangeExact = NewRange v12 NewRangeExclusive v13 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_array_dup() { + eval("def test = [1, 2, 3]"); + assert_contains_opcode("test", YARVINSN_duparray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:ArrayExact = ArrayDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_hash_dup() { + eval("def test = {a: 1, b: 2}"); + assert_contains_opcode("test", YARVINSN_duphash); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:HashExact = HashDup v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_new_hash_empty() { + eval("def test = {}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HashExact = NewHash + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_new_hash_with_elements() { + eval("def test(aval, bval) = {a: aval, b: bval}"); + assert_contains_opcode("test", YARVINSN_newhash); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :aval@0x1000 + v4:BasicObject = LoadField v2, :bval@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :aval@1 + v9:BasicObject = LoadArg :bval@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v17:StaticSymbol[:a] = Const Value(VALUE(0x1008)) + v20:StaticSymbol[:b] = Const Value(VALUE(0x1010)) + v23:HashExact = NewHash v17: v12, v20: v13 + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_string_copy() { + eval("def test = \"hello\""); + assert_contains_opcode("test", YARVINSN_dupchilledstring); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v11:StringExact = StringCopy v10 + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_bignum() { + eval("def test = 999999999999999999999999999999999999"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Bignum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_flonum() { + eval("def test = 1.5"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Flonum[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_heap_float() { + eval("def test = 1.7976931348623157e+308"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:HeapFloat[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_static_sym() { + eval("def test = :foo"); + assert_contains_opcode("test", YARVINSN_putobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StaticSymbol[:foo] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_opt_plus() { + eval("def test = 1+2"); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:Fixnum[2] = Const Value(2) + v15:BasicObject = Send v10, :+, v12 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_opt_hash_freeze() { + eval(" + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:HashExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_hash_freeze_rewritten() { + eval(" + class Hash + def freeze; 5; end + end + def test = {}.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_hash_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(HASH_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_ary_freeze() { + eval(" + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_ary_freeze_rewritten() { + eval(" + class Array + def freeze; 5; end + end + def test = [].freeze + "); + assert_contains_opcode("test", YARVINSN_opt_ary_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_freeze() { + eval(" + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_freeze_rewritten() { + eval(" + class String + def freeze; 5; end + end + def test = ''.freeze + "); + assert_contains_opcode("test", YARVINSN_opt_str_freeze); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_FREEZE)) + "); + } + + #[test] + fn test_opt_str_uminus() { + eval(" + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS) + v11:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_str_uminus_rewritten() { + eval(" + class String + def -@; 5; end + end + def test = -'' + "); + assert_contains_opcode("test", YARVINSN_opt_str_uminus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + SideExit PatchPoint(BOPRedefined(STRING_REDEFINED_OP_FLAG, BOP_UMINUS)) + "); + } + + #[test] + fn test_setlocal_getlocal() { + eval(" + def test + a = 1 + a + end + "); + assert_contains_opcodes("test", &[YARVINSN_getlocal_WC_0, YARVINSN_setlocal_WC_0]); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:Fixnum[1] = Const Value(1) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_nested_setlocal_getlocal() { + eval(" + l3 = 3 + _unused = _unused1 = nil + 1.times do |l2| + _ = nil + l2 = 2 + 1.times do |l1| + l1 = 1 + define_method(:test) do + l1 = l2 + l2 = l1 + l2 + l3 = l2 + l3 + end + end + end + "); + assert_contains_opcodes( + "test", + &[YARVINSN_getlocal_WC_1, YARVINSN_setlocal_WC_1, + YARVINSN_getlocal, YARVINSN_setlocal]); + assert_snapshot!(hir_string("test"), @" + fn block (3 levels) in <compiled>@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:CPtr = GetEP 2 + v11:BasicObject = LoadField v10, :l2@0x1000 + SetLocal :l1, l1, EP@3, v11 + v16:CPtr = GetEP 1 + v17:BasicObject = LoadField v16, :l1@0x1001 + v19:CPtr = GetEP 2 + v20:BasicObject = LoadField v19, :l2@0x1000 + v23:BasicObject = Send v17, :+, v20 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l2, l2, EP@4, v23 + v28:CPtr = GetEP 2 + v29:BasicObject = LoadField v28, :l2@0x1000 + v31:CPtr = GetEP 3 + v32:BasicObject = LoadField v31, :l3@0x1002 + v35:BasicObject = Send v29, :+, v32 # SendFallbackReason: Uncategorized(opt_plus) + SetLocal :l3, l3, EP@5, v35 + CheckInterrupts + Return v35 + " + ); + } + + #[test] + fn test_setlocal_in_default_args() { + eval(" + def test(a = (b = 1)) = [a, b] + "); + assert_contains_opcode("test", YARVINSN_setlocal_WC_0); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:NilClass = Const Value(nil) + v5:CPtr = LoadPC + v6:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v7:CBool = IsBitEqual v5, v6 + CondBranch v7, bb3(v1, v3, v4), bb6() + bb6(): + Jump bb5(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v11:BasicObject = LoadArg :self@0 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v11, v12, v13) + bb3(v20:BasicObject, v21:BasicObject, v22:NilClass): + v26:Fixnum[1] = Const Value(1) + Jump bb5(v20, v26, v26) + bb4(): + EntryPoint JIT(1) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :a@1 + v18:NilClass = Const Value(nil) + Jump bb5(v16, v17, v18) + bb5(v31:BasicObject, v32:BasicObject, v33:NilClass|Fixnum): + v39:ArrayExact = NewArray v32, v33 + CheckInterrupts + Return v39 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_tracepoint() { + eval(" + def test(a = (b = 1)) = [a, b] + TracePoint.new(:line) {}.enable + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:NilClass = Const Value(nil) + v5:CPtr = LoadPC + v6:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v7:CBool = IsBitEqual v5, v6 + CondBranch v7, bb3(v1, v3, v4), bb6() + bb6(): + Jump bb5(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v11:BasicObject = LoadArg :self@0 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v11, v12, v13) + bb3(v20:BasicObject, v21:BasicObject, v22:NilClass): + SideExit UnhandledYARVInsn(trace_putobject_INT2FIX_1_) + bb4(): + EntryPoint JIT(1) + v16:BasicObject = LoadArg :self@0 + v17:BasicObject = LoadArg :a@1 + v18:NilClass = Const Value(nil) + Jump bb5(v16, v17, v18) + bb5(v27:BasicObject, v28:BasicObject, v29:NilClass): + v35:ArrayExact = NewArray v28, v29 + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_setlocal_in_default_args_with_side_exit() { + eval(" + def test(a = (def foo = nil)) = a + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:CPtr = LoadPC + v5:CPtr[CPtr(0x1001)] = Const CPtr(0x1001) + v6:CBool = IsBitEqual v4, v5 + CondBranch v6, bb3(v1, v3), bb6() + bb6(): + Jump bb5(v1, v3) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:NilClass = Const Value(nil) + Jump bb3(v10, v11) + bb3(v17:BasicObject, v18:BasicObject): + SideExit UnhandledYARVInsn(definemethod) + bb4(): + EntryPoint JIT(1) + v14:BasicObject = LoadArg :self@0 + v15:BasicObject = LoadArg :a@1 + Jump bb5(v14, v15) + bb5(v23:BasicObject, v24:BasicObject): + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_setlocal_cyclic_default_args() { + eval(" + def test = proc { |a=a| a } + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + Jump bb3(v6, v7) + bb4(): + EntryPoint JIT(1) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + Jump bb3(v10, v11) + bb3(v13:BasicObject, v14:BasicObject): + CheckInterrupts + Return v14 + "); + } + + #[test] + fn defined_ivar() { + eval(" + def test = defined?(@foo) + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn if_defined_ivar() { + eval(" + def test + if defined?(@foo) + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_definedivar); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:TrueClass|NilClass = DefinedIvar v6, :@foo + CheckInterrupts + v13:CBool = Test v10 + v14:NilClass = RefineType v10, Falsy + CondBranch v13, bb5(), bb4(v6) + bb5(): + v16:TrueClass = RefineType v10, Truthy + v19:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v19 + bb4(v24:BasicObject): + v28:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn defined() { + eval(" + def test = return defined?(SeaChange), defined?(favourite), defined?($ruby) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:StringExact|NilClass = Defined constant, v10 + v15:StringExact|NilClass = Defined func, v6 + v17:NilClass = Const Value(nil) + v19:StringExact|NilClass = Defined global-variable, v17 + v21:ArrayExact = NewArray v12, v15, v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn defined_yield_in_method_local_iseq_returns_defined() { + eval(" + def test = defined?(yield) + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:StringExact|NilClass = Defined yield, v10 + CheckInterrupts + Return v12 + "); + } + + #[test] + fn defined_yield_in_non_method_local_iseq_returns_nil() { + eval(" + define_method(:test) { defined?(yield) } + "); + assert_contains_opcode("test", YARVINSN_defined); + assert_snapshot!(hir_string("test"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_return_const() { + eval(" + def test(cond) + if cond + 3 + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_leave); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :cond@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :cond@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb5(), bb4(v9, v17) + bb5(): + v19:Truthy = RefineType v10, Truthy + v22:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v22 + bb4(v27:BasicObject, v28:Falsy): + v32:Fixnum[4] = Const Value(4) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_merge_const() { + eval(" + def test(cond) + if cond + result = 3 + else + result = 4 + end + result + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :cond@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :cond@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + CheckInterrupts + v19:CBool = Test v12 + v20:Falsy = RefineType v12, Falsy + CondBranch v19, bb6(), bb4(v11, v20, v13) + bb6(): + v22:Truthy = RefineType v12, Truthy + v25:Fixnum[3] = Const Value(3) + CheckInterrupts + Jump bb5(v11, v22, v25) + bb4(v30:BasicObject, v31:Falsy, v32:NilClass): + v36:Fixnum[4] = Const Value(4) + Jump bb5(v30, v31, v36) + bb5(v39:BasicObject, v40:BasicObject, v41:Fixnum): + CheckInterrupts + Return v41 + "); + } + + #[test] + fn test_opt_plus_fixnum() { + eval(" + def test(a, b) = a + b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_plus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :+, v13 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_minus_fixnum() { + eval(" + def test(a, b) = a - b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_minus); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :-, v13 # SendFallbackReason: Uncategorized(opt_minus) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_mult_fixnum() { + eval(" + def test(a, b) = a * b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mult); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :*, v13 # SendFallbackReason: Uncategorized(opt_mult) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_div_fixnum() { + eval(" + def test(a, b) = a / b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_div); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :/, v13 # SendFallbackReason: Uncategorized(opt_div) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_mod_fixnum() { + eval(" + def test(a, b) = a % b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_mod); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :%, v13 # SendFallbackReason: Uncategorized(opt_mod) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_eq_fixnum() { + eval(" + def test(a, b) = a == b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_eq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :==, v13 # SendFallbackReason: Uncategorized(opt_eq) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_neq_fixnum() { + eval(" + def test(a, b) = a != b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_neq); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :!=, v13 # SendFallbackReason: Uncategorized(opt_neq) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_lt_fixnum() { + eval(" + def test(a, b) = a < b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_lt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :<, v13 # SendFallbackReason: Uncategorized(opt_lt) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_le_fixnum() { + eval(" + def test(a, b) = a <= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_le); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :<=, v13 # SendFallbackReason: Uncategorized(opt_le) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_gt_fixnum() { + eval(" + def test(a, b) = a > b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_gt); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :>, v13 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_loop() { + eval(" + def test + result = 0 + times = 10 + while times > 0 + result = result + 1 + times = times - 1 + end + result + end + test + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:NilClass = Const Value(nil) + v8:NilClass = Const Value(nil) + Jump bb3(v6, v7, v8) + bb3(v10:BasicObject, v11:NilClass, v12:NilClass): + v16:Fixnum[0] = Const Value(0) + v20:Fixnum[10] = Const Value(10) + CheckInterrupts + Jump bb5(v10, v16, v20) + bb5(v26:BasicObject, v27:BasicObject, v28:BasicObject): + v32:Fixnum[0] = Const Value(0) + v35:BasicObject = Send v28, :>, v32 # SendFallbackReason: Uncategorized(opt_gt) + CheckInterrupts + v38:CBool = Test v35 + v39:Truthy = RefineType v35, Truthy + CondBranch v38, bb4(v26, v27, v28), bb6() + bb4(v51:BasicObject, v52:BasicObject, v53:BasicObject): + v58:Fixnum[1] = Const Value(1) + v61:BasicObject = Send v52, :+, v58 # SendFallbackReason: Uncategorized(opt_plus) + v66:Fixnum[1] = Const Value(1) + v69:BasicObject = Send v53, :-, v66 # SendFallbackReason: Uncategorized(opt_minus) + Jump bb5(v51, v61, v69) + bb6(): + v41:Falsy = RefineType v35, Falsy + v43:NilClass = Const Value(nil) + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_opt_ge_fixnum() { + eval(" + def test(a, b) = a >= b + test(1, 2); test(1, 2) + "); + assert_contains_opcode("test", YARVINSN_opt_ge); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :>=, v13 # SendFallbackReason: Uncategorized(opt_ge) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_display_types() { + eval(" + def test + cond = true + if cond + 3 + else + 4 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:TrueClass = Const Value(true) + CheckInterrupts + v19:CBool[true] = Test v13 + v20 = RefineType v13, Falsy + CondBranch v19, bb5(), bb4(v8, v20) + bb5(): + v22:TrueClass = RefineType v13, Truthy + v25:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v25 + bb4(v30, v31): + v35 = Const Value(4) + CheckInterrupts + Return v35 + "); + } + + #[test] + fn test_send_without_block() { + eval(" + def bar(a, b) + a+b + end + def test + bar(2, 3) + end + "); + assert_contains_opcode("test", YARVINSN_opt_send_without_block); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:Fixnum[2] = Const Value(2) + v13:Fixnum[3] = Const Value(3) + v15:BasicObject = Send v6, :bar, v11, v13 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_send_with_block() { + eval(" + def test(a) + a.each {|item| + item + } + end + test([1,2,3]) + "); + assert_contains_opcode("test", YARVINSN_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:BasicObject = Send v10, 0x1008, :each # SendFallbackReason: Uncategorized(send) + PatchPoint NoEPEscape(test) + v18:CPtr = LoadSP + v19:BasicObject = LoadField v18, :a@0x1000 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_intern_interpolated_symbol() { + eval(r#" + def test + :"foo#{123}" + end + "#); + assert_contains_opcode("test", YARVINSN_intern); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[123] = Const Value(123) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + v21:Symbol = StringIntern v19 + CheckInterrupts + Return v21 + "); + } + + #[test] + fn different_objects_get_addresses() { + eval("def test = unknown_method([0], [1], '2', '2')"); + + // The 2 string literals have the same address because they're deduped. + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:ArrayExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:ArrayExact = ArrayDup v11 + v14:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v15:ArrayExact = ArrayDup v14 + v17:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v18:StringExact = StringCopy v17 + v20:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v21:StringExact = StringCopy v20 + v23:BasicObject = Send v6, :unknown_method, v12, v15, v18, v21 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v23 + "); + } + + #[test] + fn test_cant_compile_splat() { + eval(" + def test(a) = foo(*a) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v18:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_compile_block_arg() { + eval(" + def test(a) = foo(&a) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v9, &block, :foo, v10 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_cant_compile_kwarg() { + eval(" + def test(a) = foo(a: 1) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v17:BasicObject = Send v9, :foo, v15 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_cant_compile_kw_splat() { + eval(" + def test(a) = foo(**a) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v9, :foo, v10 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v16 + "); + } + + // TODO(max): Figure out how to generate a call with TAILCALL flag + + #[test] + fn test_compile_super() { + eval(" + def test = super() + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_compile_zsuper() { + eval(" + def test = super + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:BasicObject = InvokeSuper v6, 0x1000 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_cant_compile_super_nil_blockarg() { + eval(" + def test = super(&nil) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:NilClass = Const Value(nil) + v13:BasicObject = InvokeSuper v6, 0x1000, v11 # SendFallbackReason: Uncategorized(invokesuper) + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_compile_super_forward() { + eval(" + def test(...) = super(...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_compile_super_forward_with_block() { + eval(" + def test(...) = super { |x| x } + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + PatchPoint NoEPEscape(test) + v19:CPtr = LoadSP + v20:BasicObject = LoadField v19, :...@0x1000 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_compile_super_forward_with_use() { + eval(" + def test(...) = super(...) + 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + v18:Fixnum[1] = Const Value(1) + v21:BasicObject = Send v16, :+, v18 # SendFallbackReason: Uncategorized(opt_plus) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_compile_super_forward_with_arg() { + eval(" + def test(...) = super(1, ...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v18:BasicObject = InvokeSuperForward v9, 0x1008, v15, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized + CheckInterrupts + Return v18 + "); + } + + #[test] + fn test_compile_forwardable() { + eval("def forwardable(...) = nil"); + assert_snapshot!(hir_string("forwardable"), @" + fn forwardable@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + CheckInterrupts + Return v14 + "); + } + + // TODO(max): Figure out how to generate a call with OPT_SEND flag + + #[test] + fn test_cant_compile_kw_splat_mut() { + eval(" + def test(a) = foo **a, b: 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1008)) + v17:HashExact = NewHash + PatchPoint NoEPEscape(test) + v22:BasicObject = Send v15, :core#hash_merge_kwd, v17, v10 # SendFallbackReason: Uncategorized(opt_send_without_block) + v24:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1008)) + v27:StaticSymbol[:b] = Const Value(VALUE(0x1010)) + v29:Fixnum[1] = Const Value(1) + v31:BasicObject = Send v24, :core#hash_merge_ptr, v22, v27, v29 # SendFallbackReason: Uncategorized(opt_send_without_block) + v33:BasicObject = Send v9, :foo, v31 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_cant_compile_splat_mut() { + eval(" + def test(*) = foo *, 1 + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:ArrayExact = LoadField v2, :*@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :*@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToNewArray v10 + v18:Fixnum[1] = Const Value(1) + v20:CUInt64 = LoadField v16, :RBASIC_FLAGS@0x1001 + v21:CUInt64 = GuardNoBitsSet v20, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v16, v18 + v24:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v24 + "); + } + + #[test] + fn test_compile_forwarding() { + eval(" + def test(...) = foo(...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :...@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :...@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = SendForward v9, 0x1008, :foo, v10 # SendFallbackReason: SendForward: not yet specialized + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_compile_triple_dots_with_positional_args() { + eval(" + def test(a, ...) = foo(a, ...) + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:ArrayExact = LoadField v2, :*@0x1001 + v5:BasicObject = LoadField v2, :**@0x1002 + v6:BasicObject = LoadField v2, :&@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + v12:BasicObject = LoadArg :*@2 + v13:BasicObject = LoadArg :**@3 + v14:BasicObject = LoadArg :&@4 + v15:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v13, v14, v15) + bb3(v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:NilClass): + v29:ArrayExact = ToArray v19 + PatchPoint NoEPEscape(test) + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb4(), bb5() + bb4(): + v40:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v40, v40) + bb5(): + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) + v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v44, v21) + bb6(v34:BasicObject, v35:BasicObject): + SideExit SplatKwNotProfiled + "); + } + + #[test] + fn test_opt_new() { + eval(" + class C; end + def test = C.new + "); + assert_contains_opcode("test", YARVINSN_opt_new); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:NilClass = Const Value(nil) + v15:CBool = IsMethodCFunc v10, :new + CondBranch v15, bb6(), bb4(v6, v12, v10) + bb6(): + v17:HeapBasicObject = ObjectAlloc v10 + v19:BasicObject = Send v17, :initialize # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Jump bb5(v6, v17, v19) + bb4(v23:BasicObject, v24:NilClass, v25:BasicObject): + v28:BasicObject = Send v25, :new # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb5(v23, v28, v24) + bb5(v31:BasicObject, v32:BasicObject, v33:BasicObject): + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_opt_newarray_send_max_no_elements() { + eval(" + def test = [].max + "); + // TODO(max): Rewrite to nil + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v11:BasicObject = ArrayMax + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_newarray_send_max() { + eval(" + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX) + v20:BasicObject = ArrayMax v12, v13 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_newarray_send_max_redefined() { + eval(" + class Array + alias_method :old_max, :max + def max + old_max * 2 + end + end + + def test(a,b) = [a,b].max + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MAX)) + "); + } + + #[test] + fn test_opt_newarray_send_min_no_elements() { + eval(" + def test = [].min + "); + // TODO(max): Rewrite to nil + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MIN) + v11:BasicObject = ArrayMin + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_opt_newarray_send_min() { + eval(" + def test(a,b) = [a,b].min + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MIN) + v20:BasicObject = ArrayMin v12, v13 + CheckInterrupts + Return v20 + "); + } + + #[test] + fn test_opt_newarray_send_min_redefined() { + eval(" + class Array + alias_method :old_min, :min + def min + old_min * 2 + end + end + + def test(a,b) = [a,b].min + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_MIN)) + "); + } + + #[test] + fn test_opt_newarray_send_hash() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH) + v33:Fixnum = ArrayHash v16, v17 + PatchPoint NoEPEscape(test) + v40:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v41:ArrayExact = ArrayDup v40 + v43:BasicObject = Send v15, :puts, v41 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v33 + "); + } + + #[test] + fn test_opt_newarray_send_hash_redefined() { + eval(" + Array.class_eval { def hash = 42 } + + def test(a,b) + sum = a+b + result = [a,b].hash + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_HASH)) + "); + } + + #[test] + fn test_opt_newarray_send_pack() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].pack 'C' + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v32:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v33:StringExact = StringCopy v32 + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK) + v36:String = ArrayPackBuffer v16, v17, fmt: v33 + PatchPoint NoEPEscape(test) + v43:ArrayExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v44:ArrayExact = ArrayDup v43 + v46:BasicObject = Send v15, :puts, v44 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v36 + "); + } + + #[test] + fn test_opt_newarray_send_pack_redefined() { + eval(r#" + class Array + def pack(fmt, buffer: nil) = 5 + end + def test(a,b) + sum = a+b + result = [a,b].pack 'C' + puts [1,2,3] + result + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v32:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v33:StringExact = StringCopy v32 + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK)) + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer() { + eval(r#" + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v30:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v31:StringExact = StringCopy v30 + v37:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v38:StringExact = StringCopy v37 + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK) + v42:String = ArrayPackBuffer v16, v17, fmt: v38, buf: v31 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_opt_newarray_send_pack_buffer_redefined() { + eval(r#" + class Array + def pack(fmt, buffer: nil) = 5 + end + def test(a,b) + sum = a+b + buf = "" + [a,b].pack 'C', buffer: buf + buf + end + "#); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + v30:StringExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v31:StringExact = StringCopy v30 + v37:StringExact[VALUE(0x1010)] = Const Value(VALUE(0x1010)) + v38:StringExact = StringCopy v37 + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_PACK)) + "); + } + + #[test] + fn test_opt_newarray_send_include_p() { + eval(" + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v34:BoolExact = ArrayInclude v16, v17 | v17 + PatchPoint NoEPEscape(test) + v41:ArrayExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v42:ArrayExact = ArrayDup v41 + v44:BasicObject = Send v15, :puts, v42 # SendFallbackReason: Uncategorized(opt_send_without_block) + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_opt_newarray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + + def test(a,b) + sum = a+b + result = [a,b].include? b + puts [1,2,3] + result + end + "); + assert_contains_opcode("test", YARVINSN_opt_newarray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:10: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :a@1 + v11:BasicObject = LoadArg :b@2 + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:BasicObject, v18:NilClass, v19:NilClass): + v26:BasicObject = Send v16, :+, v17 # SendFallbackReason: Uncategorized(opt_plus) + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_duparray_send_include_p() { + eval(" + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + PatchPoint BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P) + v16:BoolExact = DupArrayInclude VALUE(0x1008) | v10 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_opt_duparray_send_include_p_redefined() { + eval(" + class Array + alias_method :old_include?, :include? + def include?(x) + old_include?(x) + end + end + def test(x) + [:a, :b].include?(x) + end + "); + assert_contains_opcode("test", YARVINSN_opt_duparray_send); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:9: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + SideExit PatchPoint(BOPRedefined(ARRAY_REDEFINED_OP_FLAG, BOP_INCLUDE_P)) + "); + } + + #[test] + fn test_opt_length() { + eval(" + def test(a,b) = [a,b].length + "); + assert_contains_opcode("test", YARVINSN_opt_length); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + v22:BasicObject = Send v19, :length # SendFallbackReason: Uncategorized(opt_length) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_opt_size() { + eval(" + def test(a,b) = [a,b].size + "); + assert_contains_opcode("test", YARVINSN_opt_size); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = NewArray v12, v13 + v22:BasicObject = Send v19, :size # SendFallbackReason: Uncategorized(opt_size) + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_getconstant() { + eval(" + def test(klass) + klass::ARGV + end + "); + assert_contains_opcode("test", YARVINSN_getconstant); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :klass@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :klass@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:FalseClass = Const Value(false) + v17:BasicObject = GetConstant v10, :ARGV, v15 + CheckInterrupts + Return v17 + "); + } + + #[test] + fn test_getinstancevariable() { + eval(" + def test = @foo + test + "); + assert_contains_opcode("test", YARVINSN_getinstancevariable); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + PatchPoint SingleRactorMode + v11:BasicObject = GetIvar v6, :@foo + CheckInterrupts + Return v11 + "); + } + + #[test] + fn test_setinstancevariable() { + eval(" + def test = @foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setinstancevariable); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + PatchPoint SingleRactorMode + SetIvar v6, :@foo, v10 + v15:HeapBasicObject = RefineType v6, HeapBasicObject + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_set_ivar_rescue_frozen() { + let result = eval(" + class Foo + attr_accessor :bar + def initialize + @bar = 1 + freeze + end + end + + def test(foo) + begin + foo.bar = 2 + rescue FrozenError + end + end + + foo = Foo.new + test(foo) + test(foo) + + foo.bar + "); + assert_eq!(VALUE::fixnum_from_usize(1), result); + } + + #[test] + fn test_getclassvariable() { + eval(" + class Foo + def self.test = @@foo + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_getclassvariable), "iseq Foo.test does not contain getclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetClassVar :@@foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setclassvariable() { + eval(" + class Foo + def self.test = @@foo = 42 + end + "); + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Foo", "test")); + assert!(iseq_contains_opcode(iseq, YARVINSN_setclassvariable), "iseq Foo.test does not contain setclassvariable"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[42] = Const Value(42) + SetClassVar :@@foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setglobal() { + eval(" + def test = $foo = 1 + test + "); + assert_contains_opcode("test", YARVINSN_setglobal); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + SetGlobal :$foo, v10 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getglobal() { + eval(" + def test = $foo + test + "); + assert_contains_opcode("test", YARVINSN_getglobal); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetGlobal :$foo + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_getblockparam() { + eval(" + def test(&block) = block + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:CPtr = GetEP 0 + v16:CUInt64 = LoadField v15, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v17:CBool = IsBlockParamModified v16 + CondBranch v17, bb4(), bb5() + bb4(): + v19:BasicObject = LoadField v15, :block@0x1002 + Jump bb6(v19) + bb5(): + v21:BasicObject = GetBlockParam :block, l0, EP@3 + Jump bb6(v21) + bb6(v14:BasicObject): + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_getblockparamproxy() { + eval(" + def test(&block) = tap(&block) + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v17:CPtr = GetEP 0 + v18:CUInt64 = LoadField v17, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v19:CBool = IsBlockParamModified v18 + CondBranch v19, bb4(), bb5() + bb4(): + v21:BasicObject = LoadField v17, :block@0x1002 + Jump bb6(v21, v21) + bb5(): + v23:CInt64 = LoadField v17, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) + v25:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v25, v10) + bb6(v15:BasicObject, v16:BasicObject): + v28:BasicObject = Send v9, &block, :tap, v15 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v28 + "); + } + + #[test] + fn test_getblockparamproxy_modified() { + eval(" + def test(&block) + b = block + tap(&block) + end + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :block@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22) + bb5(): + v24:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb6(v24) + bb6(v17:BasicObject): + v32:CPtr = GetEP 0 + v33:CUInt64 = LoadField v32, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v34:CBool = IsBlockParamModified v33 + CondBranch v34, bb7(), bb8() + bb7(): + v36:BasicObject = LoadField v32, :block@0x1002 + Jump bb9(v36, v36) + bb8(): + v38:CInt64 = LoadField v32, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v39:CInt64 = GuardAnyBitSet v38, CUInt64(1) + v40:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v40, v17) + bb9(v30:BasicObject, v31:BasicObject): + v43:BasicObject = Send v11, &block, :tap, v30 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v43 + "); + } + + #[test] + fn test_getblockparamproxy_modified_nested_block() { + eval(" + def test(&block) + proc do + b = block + tap(&block) + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v14:CPtr = GetEP 1 + v15:CUInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CBool = IsBlockParamModified v15 + CondBranch v16, bb4(), bb5() + bb4(): + v18:BasicObject = LoadField v14, :block@0x1001 + Jump bb6(v18) + bb5(): + v20:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v20) + bb6(v13:BasicObject): + v27:CPtr = GetEP 1 + v28:CUInt64 = LoadField v27, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v29:CBool = IsBlockParamModified v28 + CondBranch v29, bb7(), bb8() + bb7(): + v31:BasicObject = LoadField v27, :block@0x1001 + Jump bb9(v31) + bb8(): + v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1002 + v34:CInt64 = GuardAnyBitSet v33, CUInt64(1) + v35:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb9(v35) + bb9(v26:BasicObject): + v38:BasicObject = Send v8, &block, :tap, v26 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v38 + "); + } + + #[test] + fn test_getblockparamproxy_polymorphic_none_and_iseq() { + set_call_threshold(3); + eval(" + def test(&block) + 0.then(&block) + end + + test + test { 1 } + "); + assert_contains_opcode("test", YARVINSN_getblockparamproxy); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[0] = Const Value(0) + v18:CPtr = GetEP 0 + v19:CUInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CBool = IsBlockParamModified v19 + CondBranch v20, bb4(), bb5() + bb4(): + v22:BasicObject = LoadField v18, :block@0x1002 + Jump bb6(v22, v22) + bb5(): + v24:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1003 + v25:CInt64[1] = Const CInt64(1) + v26:CInt64 = IntAnd v24, v25 + v27:CBool = IsBitEqual v26, v25 + CondBranch v27, bb7(), bb9() + bb7(): + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v10) + bb9(): + v31:CInt64[0] = Const CInt64(0) + v32:CBool = IsBitEqual v24, v31 + CondBranch v32, bb8(), bb10() + bb8(): + v34:NilClass = Const Value(nil) + Jump bb6(v34, v10) + bb6(v16:BasicObject, v17:BasicObject): + v38:BasicObject = Send v14, &block, :then, v16 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v38 + bb10(): + SideExit BlockParamProxyProfileNotCovered + "); + } + + #[test] + fn test_getblockparam_nested_block() { + eval(" + def test(&block) + proc do + block + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v11:CPtr = GetEP 1 + v12:CUInt64 = LoadField v11, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v13:CBool = IsBlockParamModified v12 + CondBranch v13, bb4(), bb5() + bb4(): + v15:BasicObject = LoadField v11, :block@0x1001 + Jump bb6(v15) + bb5(): + v17:BasicObject = GetBlockParam :block, l1, EP@3 + Jump bb6(v17) + bb6(v10:BasicObject): + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_setblockparam() { + eval(" + def test(&block) + block = nil + end + "); + assert_contains_opcode("test", YARVINSN_setblockparam); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :block@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :block@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + SetLocal :block, l0, EP@3, v14 + v18:CPtr = GetEP 0 + v19:CInt64 = LoadField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001 + v20:CInt64[512] = Const CInt64(512) + v21:CInt64 = IntOr v19, v20 + StoreField v18, :VM_ENV_DATA_INDEX_FLAGS@0x1001, v21 + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_setblockparam_nested_block() { + eval(" + def test(&block) + proc do + block = nil + end + end + "); + assert_snapshot!(hir_string_proc("test"), @" + fn block in test@<compiled>:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:NilClass = Const Value(nil) + SetLocal :block, l1, EP@3, v10 + v14:CPtr = GetEP 1 + v15:CInt64 = LoadField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000 + v16:CInt64[512] = Const CInt64(512) + v17:CInt64 = IntOr v15, v16 + StoreField v14, :VM_ENV_DATA_INDEX_FLAGS@0x1000, v17 + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_splatkw_unprofiled_side_exits() { + eval(" + def foo(**kw, &b) = kw + def test(**kw, &b) = foo(**kw, &b) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + SideExit SplatKwNotProfiled + "); + } + + #[test] + fn test_splatkw_nil_guards_nil() { + eval(" + def foo(a, ...) = a + def test(a, ...) = foo(a, ...) + test(1) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:ArrayExact = LoadField v2, :*@0x1001 + v5:BasicObject = LoadField v2, :**@0x1002 + v6:BasicObject = LoadField v2, :&@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + v12:BasicObject = LoadArg :*@2 + v13:BasicObject = LoadArg :**@3 + v14:BasicObject = LoadArg :&@4 + v15:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v13, v14, v15) + bb3(v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:NilClass): + v29:ArrayExact = ToArray v19 + PatchPoint NoEPEscape(test) + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb4(), bb5() + bb4(): + v40:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v40, v40) + bb5(): + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v43:CInt64[0] = GuardBitEquals v42, CInt64(0) + v44:NilClass = Const Value(nil) + Jump bb6(v44, v21) + bb6(v34:BasicObject, v35:BasicObject): + v47:NilClass = GuardType v20, NilClass + v49:BasicObject = Send v17, &block, :foo, v18, v29, v47, v34 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v49 + "); + } + + #[test] + fn test_splatkw_empty_hash_guards_hash() { + eval(" + def foo(**kw, &b) = kw + def test(**kw, &b) = foo(**kw, &b) + test(&proc {}) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + v32:HashExact = GuardType v12, HashExact + v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_splatkw_hash_guards_hash() { + eval(" + def foo(**kw, &b) = kw + def test(**kw, &b) = foo(**kw, &b) + test(a: 1, &proc {}) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :b@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + v32:HashExact = GuardType v12, HashExact + v34:BasicObject = Send v11, &block, :foo, v32, v19 # SendFallbackReason: Uncategorized(send) + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_splatkw_polymorphic_side_exits() { + set_call_threshold(3); + eval(" + def foo(a, ...) = a + def test(a, ...) = foo(a, ...) + test(1) + test(1, b: 2) + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:ArrayExact = LoadField v2, :*@0x1001 + v5:BasicObject = LoadField v2, :**@0x1002 + v6:BasicObject = LoadField v2, :&@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :a@1 + v12:BasicObject = LoadArg :*@2 + v13:BasicObject = LoadArg :**@3 + v14:BasicObject = LoadArg :&@4 + v15:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v13, v14, v15) + bb3(v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:NilClass): + v29:ArrayExact = ToArray v19 + PatchPoint NoEPEscape(test) + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb4(), bb5() + bb4(): + v40:BasicObject = LoadField v36, :&@0x1005 + Jump bb6(v40, v40) + bb5(): + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v43:CInt64[0] = GuardBitEquals v42, CInt64(0) + v44:NilClass = Const Value(nil) + Jump bb6(v44, v21) + bb6(v34:BasicObject, v35:BasicObject): + SideExit SplatKwPolymorphic + "); + } + + #[test] + fn test_splatkw_with_non_hash_side_exits() { + eval(" + def foo(a:) = a + def test(obj, &block) = foo(**obj, &block) + obj = Object.new + def obj.to_hash = { a: 1 } + test(obj) { 2 } + "); + assert_contains_opcode("test", YARVINSN_splatkw); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :obj@0x1000 + v4:BasicObject = LoadField v2, :block@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :obj@1 + v9:BasicObject = LoadArg :block@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v21:CPtr = GetEP 0 + v22:CUInt64 = LoadField v21, :VM_ENV_DATA_INDEX_FLAGS@0x1002 + v23:CBool = IsBlockParamModified v22 + CondBranch v23, bb4(), bb5() + bb4(): + v25:BasicObject = LoadField v21, :block@0x1003 + Jump bb6(v25, v25) + bb5(): + v27:CInt64 = LoadField v21, :VM_ENV_DATA_INDEX_SPECVAL@0x1004 + v28:CInt64 = GuardAnyBitSet v27, CUInt64(1) + v29:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb6(v29, v13) + bb6(v19:BasicObject, v20:BasicObject): + SideExit SplatKwNotNilOrHash + "); + } + + #[test] + fn test_splatarray_mut() { + eval(" + def test(a) = [*a] + "); + assert_contains_opcode("test", YARVINSN_splatarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = ToNewArray v10 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_concattoarray() { + eval(" + def test(a) = [1, *a] + "); + assert_contains_opcode("test", YARVINSN_concattoarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:Fixnum[1] = Const Value(1) + v16:ArrayExact = NewArray v14 + v19:ArrayExact = ToArray v10 + ArrayExtend v16, v19 + CheckInterrupts + Return v16 + "); + } + + #[test] + fn test_pushtoarray_one_element() { + eval(" + def test(a) = [*a, 1] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = ToNewArray v10 + v17:Fixnum[1] = Const Value(1) + v19:CUInt64 = LoadField v15, :RBASIC_FLAGS@0x1001 + v20:CUInt64 = GuardNoBitsSet v19, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v15, v17 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_pushtoarray_multiple_elements() { + eval(" + def test(a) = [*a, 1, 2, 3] + "); + assert_contains_opcode("test", YARVINSN_pushtoarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:ArrayExact = ToNewArray v10 + v17:Fixnum[1] = Const Value(1) + v19:Fixnum[2] = Const Value(2) + v21:Fixnum[3] = Const Value(3) + v23:CUInt64 = LoadField v15, :RBASIC_FLAGS@0x1001 + v24:CUInt64 = GuardNoBitsSet v23, RUBY_FL_FREEZE=CUInt64(2048) + ArrayPush v15, v17 + ArrayPush v15, v19 + ArrayPush v15, v21 + CheckInterrupts + Return v15 + "); + } + + #[test] + fn test_aset() { + eval(" + def test(a, b) = a[b] = 1 + "); + assert_contains_opcode("test", YARVINSN_opt_aset); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v17:NilClass = Const Value(nil) + v21:Fixnum[1] = Const Value(1) + v25:BasicObject = Send v12, :[]=, v13, v21 # SendFallbackReason: Uncategorized(opt_aset) + CheckInterrupts + Return v21 + "); + } + + #[test] + fn test_aref() { + eval(" + def test(a, b) = a[b] + "); + assert_contains_opcode("test", YARVINSN_opt_aref); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + v4:BasicObject = LoadField v2, :b@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :a@1 + v9:BasicObject = LoadArg :b@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :[], v13 # SendFallbackReason: Uncategorized(opt_aref) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn opt_empty_p() { + eval(" + def test(x) = x.empty? + "); + assert_contains_opcode("test", YARVINSN_opt_empty_p); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v10, :empty? # SendFallbackReason: Uncategorized(opt_empty_p) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn opt_succ() { + eval(" + def test(x) = x.succ + "); + assert_contains_opcode("test", YARVINSN_opt_succ); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v10, :succ # SendFallbackReason: Uncategorized(opt_succ) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn opt_and() { + eval(" + def test(x, y) = x & y + "); + assert_contains_opcode("test", YARVINSN_opt_and); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :&, v13 # SendFallbackReason: Uncategorized(opt_and) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn opt_or() { + eval(" + def test(x, y) = x | y + "); + assert_contains_opcode("test", YARVINSN_opt_or); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :|, v13 # SendFallbackReason: Uncategorized(opt_or) + CheckInterrupts + Return v20 + "); + } + + #[test] + fn opt_not() { + eval(" + def test(x) = !x + "); + assert_contains_opcode("test", YARVINSN_opt_not); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:BasicObject = Send v10, :! # SendFallbackReason: Uncategorized(opt_not) + CheckInterrupts + Return v16 + "); + } + + #[test] + fn opt_regexpmatch2() { + eval(" + def test(regexp, matchee) = regexp =~ matchee + "); + assert_contains_opcode("test", YARVINSN_opt_regexpmatch2); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :regexp@0x1000 + v4:BasicObject = LoadField v2, :matchee@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :regexp@1 + v9:BasicObject = LoadArg :matchee@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v20:BasicObject = Send v12, :=~, v13 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + CheckInterrupts + Return v20 + "); + } + + #[test] + // Tests for ConstBase requires either constant or class definition, both + // of which can't be performed inside a method. + fn test_putspecialobject_vm_core_and_cbase() { + eval(" + def test + alias aliased __callee__ + end + "); + assert_contains_opcode("test", YARVINSN_putspecialobject); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:ClassSubclass[VMFrozenCore] = Const Value(VALUE(0x1000)) + v12:BasicObject = PutSpecialObject CBase + v14:StaticSymbol[:aliased] = Const Value(VALUE(0x1008)) + v16:StaticSymbol[:__callee__] = Const Value(VALUE(0x1010)) + v18:BasicObject = Send v10, :core#set_method_alias, v12, v14, v16 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v18 + "); + } + + #[test] + fn opt_reverse() { + eval(" + def reverse_odd + a, b, c = @a, @b, @c + [a, b, c] + end + + def reverse_even + a, b, c, d = @a, @b, @c, @d + [a, b, c, d] + end + "); + assert_contains_opcode("reverse_odd", YARVINSN_opt_reverse); + assert_contains_opcode("reverse_even", YARVINSN_opt_reverse); + assert_snapshot!(hir_strings!("reverse_odd", "reverse_even"), @" + fn reverse_odd@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:NilClass = Const Value(nil) + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9, v10) + bb3(v12:BasicObject, v13:NilClass, v14:NilClass, v15:NilClass): + PatchPoint SingleRactorMode + v20:BasicObject = GetIvar v12, :@a + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v12, :@b + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v12, :@c + PatchPoint NoEPEscape(reverse_odd) + v38:ArrayExact = NewArray v20, v23, v26 + CheckInterrupts + Return v38 + + fn reverse_even@<compiled>:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + v3:NilClass = Const Value(nil) + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb3(v1, v2, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:NilClass = Const Value(nil) + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11, v12) + bb3(v14:BasicObject, v15:NilClass, v16:NilClass, v17:NilClass, v18:NilClass): + PatchPoint SingleRactorMode + v23:BasicObject = GetIvar v14, :@a + PatchPoint SingleRactorMode + v26:BasicObject = GetIvar v14, :@b + PatchPoint SingleRactorMode + v29:BasicObject = GetIvar v14, :@c + PatchPoint SingleRactorMode + v32:BasicObject = GetIvar v14, :@d + PatchPoint NoEPEscape(reverse_even) + v46:ArrayExact = NewArray v23, v26, v29, v32 + CheckInterrupts + Return v46 + "); + } + + #[test] + fn test_branchnil() { + eval(" + def test(x) = x&.itself + "); + assert_contains_opcode("test", YARVINSN_branchnil); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v17:CBool = HasType v10, NilClass + v18:NilClass = Const Value(nil) + CondBranch v17, bb4(v9, v18, v18), bb5() + bb5(): + v20:NotNil = RefineType v10, NotNil + v22:BasicObject = Send v20, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + Jump bb4(v9, v20, v22) + bb4(v24:BasicObject, v25:BasicObject, v26:BasicObject): + CheckInterrupts + Return v26 + "); + } + + #[test] + fn test_infer_nilability_from_branchif() { + eval(" + def test(x) + if x + x&.itself + else + 4 + end + end + "); + assert_contains_opcode("test", YARVINSN_branchnil); + // Note that IsNil has as its operand a value that we know statically *cannot* be nil + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb6(), bb4(v9, v17) + bb6(): + v19:Truthy = RefineType v10, Truthy + CheckInterrupts + v25:CBool[false] = HasType v19, NilClass + v26:NilClass = Const Value(nil) + CondBranch v25, bb5(v9, v26, v26), bb7() + bb7(): + v28:Truthy = RefineType v19, NotNil + v30:BasicObject = Send v28, :itself # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v30 + bb4(v35:BasicObject, v36:Falsy): + v40:Fixnum[4] = Const Value(4) + Jump bb5(v35, v36, v40) + bb5(v42:BasicObject, v43:Falsy, v44:Fixnum[4]): + CheckInterrupts + Return v44 + "); + } + + #[test] + fn test_infer_truthiness_from_branch() { + eval(" + def test(x) + if x + if x + if x + 3 + else + 4 + end + else + 5 + end + else + 6 + end + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + CheckInterrupts + v16:CBool = Test v10 + v17:Falsy = RefineType v10, Falsy + CondBranch v16, bb7(), bb6(v9, v17) + bb7(): + v19:Truthy = RefineType v10, Truthy + CheckInterrupts + v24:CBool[true] = Test v19 + v25 = RefineType v19, Falsy + CondBranch v24, bb8(), bb5(v9, v25) + bb8(): + v27:Truthy = RefineType v19, Truthy + CheckInterrupts + v32:CBool[true] = Test v27 + v33 = RefineType v27, Falsy + CondBranch v32, bb9(), bb4(v9, v33) + bb9(): + v35:Truthy = RefineType v27, Truthy + v38:Fixnum[3] = Const Value(3) + CheckInterrupts + Return v38 + bb4(v63, v64): + v68 = Const Value(4) + CheckInterrupts + Return v68 + bb5(v53, v54): + v58 = Const Value(5) + CheckInterrupts + Return v58 + bb6(v43:BasicObject, v44:Falsy): + v48:Fixnum[6] = Const Value(6) + CheckInterrupts + Return v48 + "); + } + + #[test] + fn test_invokebuiltin_delegate_annotated() { + assert_contains_opcode("Float", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("Float"), @" + fn Float@<internal:kernel>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :arg@0x1000 + v4:BasicObject = LoadField v2, :exception@0x1001 + v5:BasicObject = LoadField v2, :<empty>@0x1002 + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :arg@1 + v10:BasicObject = LoadArg :exception@2 + v11:CPtr = GetEP 0 + v12:BasicObject = LoadField v11, :<empty>@0x1003 + Jump bb3(v8, v9, v10, v12) + bb3(v14:BasicObject, v15:BasicObject, v16:BasicObject, v17:BasicObject): + v21:Float = InvokeBuiltin rb_f_float, v14, v15, v16 + Jump bb4(v14, v15, v16, v17, v21) + bb4(v23:BasicObject, v24:BasicObject, v25:BasicObject, v26:BasicObject, v27:Float): + CheckInterrupts + Return v27 + "); + } + + #[test] + fn test_invokebuiltin_cexpr_annotated() { + assert_contains_opcode("class", YARVINSN_opt_invokebuiltin_delegate_leave); + assert_snapshot!(hir_string("class"), @" + fn class@<internal:kernel>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Class = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:Class): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_delegate_with_args() { + // Using an unannotated builtin to test InvokeBuiltin generation + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("Dir", "open")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate), "iseq Dir.open does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn open@<internal:dir>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :name@0x1000 + v4:BasicObject = LoadField v2, :encoding@0x1001 + v5:BasicObject = LoadField v2, :<empty>@0x1002 + v6:BasicObject = LoadField v2, :block@0x1003 + v7:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6, v7) + bb2(): + EntryPoint JIT(0) + v10:BasicObject = LoadArg :self@0 + v11:BasicObject = LoadArg :name@1 + v12:BasicObject = LoadArg :encoding@2 + v13:CPtr = GetEP 0 + v14:BasicObject = LoadField v13, :<empty>@0x1003 + v15:BasicObject = LoadArg :block@3 + v16:NilClass = Const Value(nil) + Jump bb3(v10, v11, v12, v14, v15, v16) + bb3(v18:BasicObject, v19:BasicObject, v20:BasicObject, v21:BasicObject, v22:BasicObject, v23:NilClass): + v27:BasicObject = InvokeBuiltin dir_s_open, v18, v19, v20 + PatchPoint NoEPEscape(open) + v35:CPtr = GetEP 0 + v36:CUInt64 = LoadField v35, :VM_ENV_DATA_INDEX_FLAGS@0x1004 + v37:CBool = IsBlockParamModified v36 + CondBranch v37, bb5(), bb6() + bb5(): + v39:BasicObject = LoadField v35, :block@0x1005 + Jump bb7(v39, v39) + bb6(): + v41:CInt64 = LoadField v35, :VM_ENV_DATA_INDEX_SPECVAL@0x1006 + v42:CInt64 = GuardAnyBitSet v41, CUInt64(1) + v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1008)) + Jump bb7(v43, v22) + bb7(v33:BasicObject, v34:BasicObject): + CheckInterrupts + v47:CBool = Test v33 + v48:Falsy = RefineType v33, Falsy + CondBranch v47, bb8(), bb4(v18, v19, v20, v21, v34, v27) + bb8(): + v50:Truthy = RefineType v33, Truthy + v54:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized + v57:BasicObject = InvokeBuiltin dir_s_close, v18, v27 + CheckInterrupts + Return v54 + bb4(v63:BasicObject, v64:BasicObject, v65:BasicObject, v66:BasicObject, v67:BasicObject, v68:BasicObject): + CheckInterrupts + Return v68 + "); + } + + #[test] + fn test_invokebuiltin_delegate_without_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "enable")); + assert!(iseq_contains_opcode(iseq, YARVINSN_opt_invokebuiltin_delegate_leave), "iseq GC.enable does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn enable@<internal:gc>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = InvokeBuiltin gc_enable, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:BasicObject): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invokebuiltin_with_args() { + let iseq = crate::cruby::with_rubyvm(|| get_method_iseq("GC", "start")); + assert!(iseq_contains_opcode(iseq, YARVINSN_invokebuiltin), "iseq GC.start does not contain invokebuiltin"); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn start@<internal:gc>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :full_mark@0x1000 + v4:BasicObject = LoadField v2, :immediate_mark@0x1001 + v5:BasicObject = LoadField v2, :immediate_sweep@0x1002 + v6:BasicObject = LoadField v2, :<empty>@0x1003 + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :full_mark@1 + v11:BasicObject = LoadArg :immediate_mark@2 + v12:BasicObject = LoadArg :immediate_sweep@3 + v13:CPtr = GetEP 0 + v14:BasicObject = LoadField v13, :<empty>@0x1004 + Jump bb3(v9, v10, v11, v12, v14) + bb3(v16:BasicObject, v17:BasicObject, v18:BasicObject, v19:BasicObject, v20:BasicObject): + v27:FalseClass = Const Value(false) + v29:BasicObject = InvokeBuiltin gc_start_internal, v16, v17, v18, v19, v27 + CheckInterrupts + Return v29 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_name() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "name")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn name@<internal:symbol>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn test_invoke_leaf_builtin_symbol_to_s() { + let iseq = crate::cruby::with_rubyvm(|| get_instance_method_iseq("Symbol", "to_s")); + let function = iseq_to_hir(iseq).unwrap(); + assert_snapshot!(hir_string_function(&function), @" + fn to_s@<internal:symbol>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact = InvokeBuiltin leaf <inline_expr>, v6 + Jump bb4(v6, v10) + bb4(v12:BasicObject, v13:StringExact): + CheckInterrupts + Return v13 + "); + } + + #[test] + fn dupn() { + eval(" + def test(x) = (x[0, 1] ||= 2) + "); + assert_contains_opcode("test", YARVINSN_dupn); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :x@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v14:NilClass = Const Value(nil) + v17:Fixnum[0] = Const Value(0) + v19:Fixnum[1] = Const Value(1) + v22:BasicObject = Send v10, :[], v17, v19 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + v26:CBool = Test v22 + v27:Truthy = RefineType v22, Truthy + CondBranch v26, bb4(v9, v10, v14, v10, v17, v19, v27), bb5() + bb4(v41:BasicObject, v42:BasicObject, v43:NilClass, v44:BasicObject, v45:Fixnum[0], v46:Fixnum[1], v47:Truthy): + CheckInterrupts + Return v47 + bb5(): + v29:Falsy = RefineType v22, Falsy + v32:Fixnum[2] = Const Value(2) + v35:BasicObject = Send v10, :[]=, v17, v19, v32 # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v32 + "); + } + + #[test] + fn test_objtostring_anytostring() { + eval(" + def test = \"#{1}\" + "); + assert_contains_opcode("test", YARVINSN_objtostring); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:Fixnum[1] = Const Value(1) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_string_concat() { + eval(r##" + def test = "#{1}#{2}#{3}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:StringExact = StringConcat v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_string_concat_empty() { + eval(r##" + def test = "#{}" + "##); + assert_contains_opcode("test", YARVINSN_concatstrings); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v12:NilClass = Const Value(nil) + v15:BasicObject = ObjToString v12 + v17:String = AnyToString v12, str: v15 + v19:StringExact = StringConcat v10, v17 + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_toregexp() { + eval(r##" + def test = /#{1}#{2}#{3}/ + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:Fixnum[3] = Const Value(3) + v27:BasicObject = ObjToString v24 + v29:String = AnyToString v24, str: v27 + v31:RegexpExact = ToRegexp v15, v22, v29 + CheckInterrupts + Return v31 + "); + } + + #[test] + fn test_toregexp_with_options() { + eval(r##" + def test = /#{1}#{2}/mixn + "##); + assert_contains_opcode("test", YARVINSN_toregexp); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v13:BasicObject = ObjToString v10 + v15:String = AnyToString v10, str: v13 + v17:Fixnum[2] = Const Value(2) + v20:BasicObject = ObjToString v17 + v22:String = AnyToString v17, str: v20 + v24:RegexpExact = ToRegexp v15, v22, MULTILINE|IGNORECASE|EXTENDED|NOENCODING + CheckInterrupts + Return v24 + "); + } + + #[test] + fn throw() { + eval(" + define_method(:throw_return) { return 1 } + define_method(:throw_break) { break 2 } + "); + assert_contains_opcode("throw_return", YARVINSN_throw); + assert_contains_opcode("throw_break", YARVINSN_throw); + assert_snapshot!(hir_strings!("throw_return", "throw_break"), @" + fn block in <compiled>@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v12:Fixnum[1] = Const Value(1) + Throw TAG_RETURN, v12 + + fn block in <compiled>@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v12:Fixnum[2] = Const Value(2) + Throw TAG_BREAK, v12 + "); + } + + #[test] + fn test_invokeblock() { + eval(r#" + def test + yield + end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = InvokeBlock # SendFallbackReason: InvokeBlock: not yet specialized + CheckInterrupts + Return v10 + "); + } + + #[test] + fn test_invokeblock_with_args() { + eval(r#" + def test(x, y) + yield x, y + end + "#); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :x@0x1000 + v4:BasicObject = LoadField v2, :y@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :x@1 + v9:BasicObject = LoadArg :y@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:BasicObject = InvokeBlock, v12, v13 # SendFallbackReason: InvokeBlock: not yet specialized + CheckInterrupts + Return v19 + "); + } + + #[test] + fn test_expandarray_no_splat() { + eval(r#" + def test(o) + a, b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :o@1 + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:NilClass, v16:NilClass): + v22:ArrayExact = GuardType v14, ArrayExact + v23:CInt64 = ArrayLength v22 + v24:CInt64[2] = Const CInt64(2) + v25:CInt64 = GuardGreaterEq v23, v24 + v26:CInt64[1] = Const CInt64(1) + v27:BasicObject = ArrayAref v22, v26 + v28:CInt64[0] = Const CInt64(0) + v29:BasicObject = ArrayAref v22, v28 + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v14 + "); + } + + #[test] + fn test_expandarray_splat() { + eval(r#" + def test(o) + a, *b = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5) + bb2(): + EntryPoint JIT(0) + v8:BasicObject = LoadArg :self@0 + v9:BasicObject = LoadArg :o@1 + v10:NilClass = Const Value(nil) + v11:NilClass = Const Value(nil) + Jump bb3(v8, v9, v10, v11) + bb3(v13:BasicObject, v14:BasicObject, v15:NilClass, v16:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_expandarray_splat_post() { + eval(r#" + def test(o) + a, *b, c = o + end + "#); + assert_contains_opcode("test", YARVINSN_expandarray); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :o@0x1000 + v4:NilClass = Const Value(nil) + v5:NilClass = Const Value(nil) + v6:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4, v5, v6) + bb2(): + EntryPoint JIT(0) + v9:BasicObject = LoadArg :self@0 + v10:BasicObject = LoadArg :o@1 + v11:NilClass = Const Value(nil) + v12:NilClass = Const Value(nil) + v13:NilClass = Const Value(nil) + Jump bb3(v9, v10, v11, v12, v13) + bb3(v15:BasicObject, v16:BasicObject, v17:NilClass, v18:NilClass, v19:NilClass): + SideExit UnhandledYARVInsn(expandarray) + "); + } + + #[test] + fn test_checkkeyword_tests_fixnum_bit() { + eval(r#" + def test(kw: 1 + 1) = kw + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :kw@0x1000 + v4:BasicObject = LoadField v2, :<empty>@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :kw@1 + v9:CPtr = GetEP 0 + v10:BasicObject = LoadField v9, :<empty>@0x1002 + Jump bb3(v7, v8, v10) + bb3(v12:BasicObject, v13:BasicObject, v14:BasicObject): + v17:BoolExact = FixnumBitCheck v14, 0 + CheckInterrupts + v20:CBool = Test v17 + v21:TrueClass = RefineType v17, Truthy + CondBranch v20, bb4(v12, v13, v14), bb5() + bb5(): + v23:FalseClass = RefineType v17, Falsy + v25:Fixnum[1] = Const Value(1) + v27:Fixnum[1] = Const Value(1) + v30:BasicObject = Send v25, :+, v27 # SendFallbackReason: Uncategorized(opt_plus) + Jump bb4(v12, v30, v14) + bb4(v33:BasicObject, v34:BasicObject, v35:BasicObject): + CheckInterrupts + Return v34 + "); + } + + #[test] + fn test_checkkeyword_too_many_keywords_causes_side_exit() { + eval(r#" + def test(k1: k1, k2: k2, k3: k3, k4: k4, k5: k5, + k6: k6, k7: k7, k8: k8, k9: k9, k10: k10, k11: k11, + k12: k12, k13: k13, k14: k14, k15: k15, k16: k16, + k17: k17, k18: k18, k19: k19, k20: k20, k21: k21, + k22: k22, k23: k23, k24: k24, k25: k25, k26: k26, + k27: k27, k28: k28, k29: k29, k30: k30, k31: k31, + k32: k32, k33: k33) = k1 + "#); + assert_contains_opcode("test", YARVINSN_checkkeyword); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :k1@0x1000 + v4:BasicObject = LoadField v2, :k2@0x1001 + v5:BasicObject = LoadField v2, :k3@0x1002 + v6:BasicObject = LoadField v2, :k4@0x1003 + v7:BasicObject = LoadField v2, :k5@0x1004 + v8:BasicObject = LoadField v2, :k6@0x1005 + v9:BasicObject = LoadField v2, :k7@0x1006 + v10:BasicObject = LoadField v2, :k8@0x1007 + v11:BasicObject = LoadField v2, :k9@0x1008 + v12:BasicObject = LoadField v2, :k10@0x1009 + v13:BasicObject = LoadField v2, :k11@0x100a + v14:BasicObject = LoadField v2, :k12@0x100b + v15:BasicObject = LoadField v2, :k13@0x100c + v16:BasicObject = LoadField v2, :k14@0x100d + v17:BasicObject = LoadField v2, :k15@0x100e + v18:BasicObject = LoadField v2, :k16@0x100f + v19:BasicObject = LoadField v2, :k17@0x1010 + v20:BasicObject = LoadField v2, :k18@0x1011 + v21:BasicObject = LoadField v2, :k19@0x1012 + v22:BasicObject = LoadField v2, :k20@0x1013 + v23:BasicObject = LoadField v2, :k21@0x1014 + v24:BasicObject = LoadField v2, :k22@0x1015 + v25:BasicObject = LoadField v2, :k23@0x1016 + v26:BasicObject = LoadField v2, :k24@0x1017 + v27:BasicObject = LoadField v2, :k25@0x1018 + v28:BasicObject = LoadField v2, :k26@0x1019 + v29:BasicObject = LoadField v2, :k27@0x101a + v30:BasicObject = LoadField v2, :k28@0x101b + v31:BasicObject = LoadField v2, :k29@0x101c + v32:BasicObject = LoadField v2, :k30@0x101d + v33:BasicObject = LoadField v2, :k31@0x101e + v34:BasicObject = LoadField v2, :k32@0x101f + v35:BasicObject = LoadField v2, :k33@0x1020 + v36:BasicObject = LoadField v2, :<empty>@0x1021 + Jump bb3(v1, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36) + bb2(): + EntryPoint JIT(0) + v39:BasicObject = LoadArg :self@0 + v40:BasicObject = LoadArg :k1@1 + v41:BasicObject = LoadArg :k2@2 + v42:BasicObject = LoadArg :k3@3 + v43:BasicObject = LoadArg :k4@4 + v44:BasicObject = LoadArg :k5@5 + v45:BasicObject = LoadArg :k6@6 + v46:BasicObject = LoadArg :k7@7 + v47:BasicObject = LoadArg :k8@8 + v48:BasicObject = LoadArg :k9@9 + v49:BasicObject = LoadArg :k10@10 + v50:BasicObject = LoadArg :k11@11 + v51:BasicObject = LoadArg :k12@12 + v52:BasicObject = LoadArg :k13@13 + v53:BasicObject = LoadArg :k14@14 + v54:BasicObject = LoadArg :k15@15 + v55:BasicObject = LoadArg :k16@16 + v56:BasicObject = LoadArg :k17@17 + v57:BasicObject = LoadArg :k18@18 + v58:BasicObject = LoadArg :k19@19 + v59:BasicObject = LoadArg :k20@20 + v60:BasicObject = LoadArg :k21@21 + v61:BasicObject = LoadArg :k22@22 + v62:BasicObject = LoadArg :k23@23 + v63:BasicObject = LoadArg :k24@24 + v64:BasicObject = LoadArg :k25@25 + v65:BasicObject = LoadArg :k26@26 + v66:BasicObject = LoadArg :k27@27 + v67:BasicObject = LoadArg :k28@28 + v68:BasicObject = LoadArg :k29@29 + v69:BasicObject = LoadArg :k30@30 + v70:BasicObject = LoadArg :k31@31 + v71:BasicObject = LoadArg :k32@32 + v72:BasicObject = LoadArg :k33@33 + v73:CPtr = GetEP 0 + v74:BasicObject = LoadField v73, :<empty>@0x1022 + Jump bb3(v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49, v50, v51, v52, v53, v54, v55, v56, v57, v58, v59, v60, v61, v62, v63, v64, v65, v66, v67, v68, v69, v70, v71, v72, v74) + bb3(v76:BasicObject, v77:BasicObject, v78:BasicObject, v79:BasicObject, v80:BasicObject, v81:BasicObject, v82:BasicObject, v83:BasicObject, v84:BasicObject, v85:BasicObject, v86:BasicObject, v87:BasicObject, v88:BasicObject, v89:BasicObject, v90:BasicObject, v91:BasicObject, v92:BasicObject, v93:BasicObject, v94:BasicObject, v95:BasicObject, v96:BasicObject, v97:BasicObject, v98:BasicObject, v99:BasicObject, v100:BasicObject, v101:BasicObject, v102:BasicObject, v103:BasicObject, v104:BasicObject, v105:BasicObject, v106:BasicObject, v107:BasicObject, v108:BasicObject, v109:BasicObject, v110:BasicObject): + SideExit TooManyKeywordParameters + "); + } + + #[test] + fn test_array_each() { + assert_snapshot!(hir_string_proc("Array.instance_method(:each)"), @" + fn each@<internal:array>: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:NilClass = Const Value(nil) + Jump bb3(v1, v2) + bb2(): + EntryPoint JIT(0) + v5:BasicObject = LoadArg :self@0 + v6:NilClass = Const Value(nil) + Jump bb3(v5, v6) + bb3(v8:BasicObject, v9:NilClass): + v13:NilClass = Const Value(nil) + v15:TrueClass|NilClass = Defined yield, v13 + v17:CBool = Test v15 + v18:NilClass = RefineType v15, Falsy + CondBranch v17, bb9(), bb4(v8, v9) + bb9(): + v20:TrueClass = RefineType v15, Truthy + Jump bb6(v8, v9) + bb6(v30:BasicObject, v31:NilClass): + v35:Fixnum[0] = Const Value(0) + Jump bb8(v30, v35) + bb8(v48:BasicObject, v49:Fixnum): + v52:BoolExact = InvokeBuiltin rb_jit_ary_at_end, v48, v49 + v54:CBool = Test v52 + v55:FalseClass = RefineType v52, Falsy + CondBranch v54, bb10(), bb7(v48, v49) + bb10(): + v57:TrueClass = RefineType v52, Truthy + v59:NilClass = Const Value(nil) + CheckInterrupts + Return v48 + bb7(v67:BasicObject, v68:Fixnum): + v72:BasicObject = InvokeBuiltin rb_jit_ary_at, v67, v68 + v74:BasicObject = InvokeBlock, v72 # SendFallbackReason: InvokeBlock: not yet specialized + v78:Fixnum = InvokeBuiltin rb_jit_fixnum_inc, v67, v68 + PatchPoint NoEPEscape(each) + Jump bb8(v67, v78) + bb4(v23:BasicObject, v24:NilClass): + v28:BasicObject = InvokeBuiltin <inline_expr>, v23 + Jump bb5(v23, v24, v28) + bb5(v40:BasicObject, v41:NilClass, v42:BasicObject): + CheckInterrupts + Return v42 + "); + } + + #[test] + fn test_induce_side_exit() { + eval(" + class NonTopLexicalScope + RubyVM = 0 + def test + RubyVM::ZJIT.induce_side_exit! # lexical scope dependant -- should not recognize + ::RubyVM::ZJIT.induce_side_exit! + end + end + "); + assert_snapshot!(hir_string_proc("NonTopLexicalScope.instance_method(:test)"), @" + fn test@<compiled>:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_side_exit! # SendFallbackReason: Uncategorized(opt_send_without_block) + v16:BasicObject = GetConstantPath 0x1000 + SideExit DirectiveInduced + "); + } + + #[test] + fn test_induce_side_exit_sensitive_to_constant_state() { + eval(" + def test = ::RubyVM::ZJIT.induce_side_exit! + "); + assert!(hir_string("test").contains("SideExit DirectiveInduced")); + eval(" + class RubyVM + remove_const(:ZJIT) + end + "); + let hir_after_removal = hir_string("test"); + assert_eq!(false, hir_string("test").contains("SideExit DirectiveInduced"), "should not work when the constant lookup would fail"); + assert_snapshot!(hir_after_removal, @" + fn test@<compiled>:2: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_side_exit! # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_induce_side_exit_doesnt_work_when_method_after_undef() { + eval(" + class << RubyVM::ZJIT + undef :induce_side_exit! + end + def test = ::RubyVM::ZJIT.induce_side_exit! + "); + assert_eq!(false, hir_string("test").contains("SideExit DirectiveInduced"), "should not work after undef"); + } + + #[test] + fn test_induce_compile_failure_does_not_trigger_autoload() { + eval(" + class RubyVM + remove_const(:ZJIT) + autoload :ZJIT, 'a-file-that-does-not-exist-as-a-trap' + end + def test = ::RubyVM::ZJIT.induce_compile_failure! + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:6: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_compile_failure! # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_induce_compile_failure_checks_full_const_path() { + eval("def test = ::RubyVM::ZJIT::TooDeep.induce_compile_failure!"); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:1: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:BasicObject = GetConstantPath 0x1000 + v12:BasicObject = Send v10, :induce_compile_failure! # SendFallbackReason: Uncategorized(opt_send_without_block) + CheckInterrupts + Return v12 + "); + } + + #[test] + fn test_induce_compile_failure() { + eval("def test = ::RubyVM::ZJIT.induce_compile_failure!"); + assert_compile_fails("test", ParseError::DirectiveInduced); + } + + #[test] + fn test_induce_breakpoint() { + eval("def test = ::RubyVM::ZJIT.induce_breakpoint!"); + assert!(hir_string("test").contains("BreakPoint")); + } + + #[test] + fn test_induce_breakpoint_returns_nil() { + eval(" + def test + x = ::RubyVM::ZJIT.induce_breakpoint! + x + end + "); + let hir = hir_string("test"); + assert!(hir.contains("BreakPoint")); + assert!(hir.contains("Return v")); + } + + #[test] + fn test_getspecialnumber() { + eval(" + def test(a) + a =~/(hello)/ + $1 + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:RegexpExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v18:BasicObject = Send v10, :=~, v15 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + v22:StringExact|NilClass = GetSpecialNumber 2 + CheckInterrupts + Return v22 + "); + } + + #[test] + fn test_getspecialsymbol() { + eval(" + def test(a) + a =~/(hello)/ + $& + end + "); + assert_snapshot!(hir_string("test"), @" + fn test@<compiled>:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :a@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :a@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:RegexpExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v18:BasicObject = Send v10, :=~, v15 # SendFallbackReason: Uncategorized(opt_regexpmatch2) + v22:StringExact|NilClass = GetSpecialSymbol LastMatch + CheckInterrupts + Return v22 + "); + } +} + + /// Test successor and predecessor set computations. + #[cfg(test)] + mod control_flow_info_tests { + use super::*; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb1, bb2)); + assert!(cfi.is_succeeded_by(bb2, bb1)); + assert!(cfi.predecessors(bb3).eq([bb2])); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: v1, if_true: edge(bb2), if_false: edge(bb1) }); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + + assert!(cfi.is_preceded_by(bb2, bb3)); + assert!(cfi.is_preceded_by(bb1, bb3)); + assert!(!cfi.is_preceded_by(bb0, bb3)); + assert!(cfi.is_succeeded_by(bb1, bb0)); + assert!(cfi.is_succeeded_by(bb3, bb1)); + } + + #[test] + fn test_cfi_deduplicated_successors_and_predecessors() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + // Construct two separate jump instructions. + let v1 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: v1, if_true: edge(bb1), if_false: edge(bb1)}); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + + assert_eq!(cfi.predecessors(bb1).collect::<Vec<_>>().len(), 1); + assert_eq!(cfi.successors(bb0).collect::<Vec<_>>().len(), 1); + } + } + + /// Test dominator set computations. + #[cfg(test)] + mod dom_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + fn assert_dominators_contains_self(function: &Function, dominators: &Dominators) { + for (i, _) in function.blocks.iter().enumerate() { + // Ensure that each dominating set contains the block itself. + assert!(dominators.is_dominated_by(BlockId(i), BlockId(i))); + } + } + + #[test] + fn test_linked_list() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + function.push_insn(bb1, Insn::Jump(edge(bb2))); + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + Jump bb4() + bb4(): + v3:Any = Const CBool(true) + Return v3 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb1, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb1, bb2, bb3]); + } + + #[test] + fn test_diamond() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + let val = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val, if_true: edge(bb1), if_false: edge(bb2) }); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + function.push_insn(bb1, Insn::Jump(edge(bb3))); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + CondBranch v0, bb2(), bb3() + bb2(): + Jump bb4() + bb3(): + Jump bb4() + bb4(): + v4:Any = Const CBool(true) + Return v4 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb3]); + } + + #[test] + fn test_complex_cfg() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + let bb7 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let v0 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::CondBranch { val: v0, if_true: edge(bb2), if_false: edge(bb4) }); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + let v1 = function.push_insn(bb3, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb3, Insn::CondBranch { val: v1, if_true: edge(bb5), if_false: edge(bb7) }); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + function.push_insn(bb5, Insn::Jump(edge(bb6))); + + function.push_insn(bb6, Insn::Jump(edge(bb7))); + + let retval = function.push_insn(bb7, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb7, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + v1:Any = Const Value(false) + CondBranch v1, bb3(), bb5() + bb3(): + Jump bb4() + bb4(): + v4:Any = Const Value(false) + CondBranch v4, bb6(), bb8() + bb5(): + Jump bb6() + bb6(): + Jump bb7() + bb7(): + Jump bb8() + bb8(): + v9:Any = Const CBool(true) + Return v9 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb1, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb1, bb2, bb3]); + assert_eq!(dominators.dominators(bb4), vec![entries, bb0, bb1, bb4]); + assert_eq!(dominators.dominators(bb5), vec![entries, bb0, bb1, bb5]); + assert_eq!(dominators.dominators(bb6), vec![entries, bb0, bb1, bb5, bb6]); + assert_eq!(dominators.dominators(bb7), vec![entries, bb0, bb1, bb7]); + } + + #[test] + fn test_back_edges() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + + let v0 = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: v0, if_true: edge(bb1), if_false: edge(bb4) }); + + let v1 = function.push_insn(bb1, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb1, Insn::CondBranch { val: v1, if_true: edge(bb2), if_false: edge(bb3) }); + + function.push_insn(bb2, Insn::Jump(edge(bb3))); + + function.push_insn(bb4, Insn::Jump(edge(bb5))); + + let v2 = function.push_insn(bb5, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb5, Insn::CondBranch { val: v2, if_true: edge(bb3), if_false: edge(bb4) }); + + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb3, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + CondBranch v0, bb2(), bb5() + bb2(): + v2:Any = Const Value(false) + CondBranch v2, bb3(), bb4() + bb3(): + Jump bb4() + bb5(): + Jump bb6() + bb6(): + v6:Any = Const Value(false) + CondBranch v6, bb4(), bb5() + bb4(): + v8:Any = Const CBool(true) + Return v8 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb0, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb0, bb1, bb2]); + assert_eq!(dominators.dominators(bb3), vec![entries, bb0, bb3]); + assert_eq!(dominators.dominators(bb4), vec![entries, bb0, bb4]); + assert_eq!(dominators.dominators(bb5), vec![entries, bb0, bb4, bb5]); + } + + #[test] + fn test_multiple_entry_blocks() { + let mut function = Function::new(std::ptr::null()); + + let entries = function.entries_block; + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + function.jit_entry_blocks.push(bb1); + let bb2 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb2, Insn::Return { val: retval }); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb3() + bb2(): + Jump bb3() + bb3(): + v2:Any = Const CBool(true) + Return v2 + "); + + let dominators = Dominators::new(&function); + assert_dominators_contains_self(&function, &dominators); + + assert_eq!(dominators.dominators(bb0), vec![entries, bb0]); + assert_eq!(dominators.dominators(bb1), vec![entries, bb1]); + assert_eq!(dominators.dominators(bb2), vec![entries, bb2]); + + assert!(!dominators.is_dominated_by(bb1, bb2)); + } + } + + /// Test loop information computation. +#[cfg(test)] +mod loop_info_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_loop_depth() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ ┌─────┐ + // ┌►│ bb2 ├─────►│ bb1 │ + // │ └──┬──┘ T └──┬──┘ + // │ F│ │ + // │ ┌──▼──┐ │ + // │ │ bb3 │ │ + // │ └─────┘ │ + // └─────────────────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::CondBranch { val, if_true: edge(bb1), if_false: edge(bb3) }); + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb3, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb3() + bb3(): + v1:Any = Const Value(false) + CondBranch v1, bb2(), bb4() + bb2(): + Jump bb3() + bb4(): + v3:Any = Const CBool(true) + Return v3 + "); + + assert!(loop_info.is_loop_header(bb2)); + assert!(loop_info.is_back_edge_source(bb1)); + assert_eq!(loop_info.loop_depth(bb1), 1); + } + + #[test] + fn test_nested_loops() { + // ┌─────┐ + // │ bb0 ◄─────┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄───┐ │ + // └──┬──┘ │ │ + // │ │ │ + // ┌──▼──┐ │ │ + // │ bb2 ┼───┘ │ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb3 ┼─────┘ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb4 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let cond = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb3) }); + + let cond = function.push_insn(bb3, Insn::Const { val: Const::Value(Qtrue) }); + let _ = function.push_insn(bb3, Insn::CondBranch { val: cond, if_true: edge(bb0), if_false: edge(bb4) }); + + let retval = function.push_insn(bb4, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb4, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + v2:Any = Const Value(false) + CondBranch v2, bb2(), bb4() + bb4(): + v4:Any = Const Value(true) + CondBranch v4, bb1(), bb5() + bb5(): + v6:Any = Const CBool(true) + Return v6 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 1); + assert_eq!(loop_info.loop_depth(bb4), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + } + + #[test] + fn test_complex_loops() { + // ┌─────┐ + // ┌──────► bb0 │ + // │ └──┬──┘ + // │ ┌────┴────┐ + // │ ┌──▼──┐ ┌──▼──┐ + // │ │ bb1 ◄─┐ │ bb3 ◄─┐ + // │ └──┬──┘ │ └──┬──┘ │ + // │ │ │ │ │ + // │ ┌──▼──┐ │ ┌──▼──┐ │ + // │ │ bb2 ┼─┘ │ bb4 ┼─┘ + // │ └──┬──┘ └──┬──┘ + // │ └────┬────┘ + // │ ┌──▼──┐ + // └──────┼ bb5 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb6 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb3) }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let _ = function.push_insn(bb2, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb5) }); + + function.push_insn(bb3, Insn::Jump(edge(bb4))); + + let _ = function.push_insn(bb4, Insn::CondBranch { val: cond, if_true: edge(bb3), if_false: edge(bb5) }); + + let _ = function.push_insn(bb5, Insn::CondBranch { val: cond, if_true: edge(bb0), if_false: edge(bb6) }); + + let retval = function.push_insn(bb6, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb6, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + CondBranch v0, bb2(), bb4() + bb2(): + Jump bb3() + bb3(): + CondBranch v0, bb2(), bb6() + bb4(): + Jump bb5() + bb5(): + CondBranch v0, bb4(), bb6() + bb6(): + CondBranch v0, bb1(), bb7() + bb7(): + v7:Any = Const CBool(true) + Return v7 + "); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + assert!(loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb5)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb6)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 2); + assert_eq!(loop_info.loop_depth(bb3), 2); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + assert_eq!(loop_info.loop_depth(bb6), 0); + + assert!(loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + } + + #[test] + fn linked_list_non_loop() { + // ┌─────┐ + // │ bb0 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb1 │ + // └──┬──┘ + // │ + // ┌──▼──┐ + // │ bb2 │ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + + let retval = function.push_insn(bb2, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb2, Insn::Return { val: retval }); + + function.seal_entries(); + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + Jump bb2() + bb2(): + Jump bb3() + bb3(): + v2:Any = Const CBool(true) + Return v2 + "); + + assert!(!loop_info.is_loop_header(bb0)); + assert!(!loop_info.is_loop_header(bb1)); + assert!(!loop_info.is_loop_header(bb2)); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + + assert_eq!(loop_info.loop_depth(bb0), 0); + assert_eq!(loop_info.loop_depth(bb1), 0); + assert_eq!(loop_info.loop_depth(bb2), 0); + } + + #[test] + fn triple_nested_loop() { + // ┌─────┐ + // │ bb0 ◄──┐ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb1 ◄─┐│ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb2 ◄┐││ + // └──┬──┘│││ + // │ │││ + // ┌──▼──┐│││ + // │ bb3 ┼┘││ + // └──┬──┘ ││ + // │ ││ + // ┌──▼──┐ ││ + // │ bb4 ┼─┘│ + // └──┬──┘ │ + // │ │ + // ┌──▼──┐ │ + // │ bb5 ┼──┘ + // └─────┘ + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + let bb4 = function.new_block(0); + let bb5 = function.new_block(0); + let bb6 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb0, Insn::Jump(edge(bb1))); + let _ = function.push_insn(bb1, Insn::Jump(edge(bb2))); + let _ = function.push_insn(bb2, Insn::Jump(edge(bb3))); + let _ = function.push_insn(bb3, Insn::CondBranch {val: cond, if_true: edge(bb2), if_false: edge(bb4) }); + let _ = function.push_insn(bb4, Insn::CondBranch {val: cond, if_true: edge(bb1), if_false: edge(bb5) }); + let _ = function.push_insn(bb5, Insn::CondBranch {val: cond, if_true: edge(bb0), if_false: edge(bb6) }); + function.push_insn(bb6, Insn::Unreachable); + + function.seal_entries(); + assert_snapshot!(format!("{}", FunctionPrinter::without_snapshot(&function)), @" + fn <manual>: + bb1(): + v0:Any = Const Value(false) + Jump bb2() + bb2(): + Jump bb3() + bb3(): + Jump bb4() + bb4(): + CondBranch v0, bb3(), bb5() + bb5(): + CondBranch v0, bb2(), bb6() + bb6(): + CondBranch v0, bb1(), bb7() + bb7(): + Unreachable + "); + + let cfi = ControlFlowInfo::new(&function); + let dominators = Dominators::new(&function); + let loop_info = LoopInfo::new(&cfi, &dominators); + + assert!(!loop_info.is_back_edge_source(bb0)); + assert!(!loop_info.is_back_edge_source(bb1)); + assert!(!loop_info.is_back_edge_source(bb2)); + assert!(loop_info.is_back_edge_source(bb3)); + assert!(loop_info.is_back_edge_source(bb4)); + assert!(loop_info.is_back_edge_source(bb5)); + + assert_eq!(loop_info.loop_depth(bb0), 1); + assert_eq!(loop_info.loop_depth(bb1), 2); + assert_eq!(loop_info.loop_depth(bb2), 3); + assert_eq!(loop_info.loop_depth(bb3), 3); + assert_eq!(loop_info.loop_depth(bb4), 2); + assert_eq!(loop_info.loop_depth(bb5), 1); + + assert!(loop_info.is_loop_header(bb0)); + assert!(loop_info.is_loop_header(bb1)); + assert!(loop_info.is_loop_header(bb2)); + assert!(!loop_info.is_loop_header(bb3)); + assert!(!loop_info.is_loop_header(bb4)); + assert!(!loop_info.is_loop_header(bb5)); + } + } + +/// Test dumping to iongraph format. +#[cfg(test)] +mod iongraph_tests { + use super::*; + use insta::assert_snapshot; + + fn edge(target: BlockId) -> BranchEdge { + BranchEdge { target, args: vec![] } + } + + #[test] + fn test_simple_function() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let retval = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: retval }); + function.seal_entries(); + + let json = function.to_iongraph_pass("simple"); + assert_snapshot!(json.to_string(), @r#"{"name":"simple", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4098, "id":2, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_two_blocks() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb1))); + + let retval = function.push_insn(bb1, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb1, Insn::Return { val: retval }); + + function.seal_entries(); + let json = function.to_iongraph_pass("two_blocks"); + assert_snapshot!(json.to_string(), @r#"{"name":"two_blocks", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4099, "id":3, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[2], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb2()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4097, "id":1, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4098, "id":2, "opcode":"Return v1", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_instructions() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + + let val1 = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::Return { val: val1 }); + + function.seal_entries(); + let json = function.to_iongraph_pass("multiple_instructions"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_instructions", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4098, "id":2, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"Return v0", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_conditional_branch() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb2) }); + + let retval1 = function.push_insn(bb2, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb2, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval2 }); + + function.seal_entries(); + let json = function.to_iongraph_pass("conditional_branch"); + assert_snapshot!(json.to_string(), @r#"{"name":"conditional_branch", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4102, "id":6, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[2, 3], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"CondBranch v0, bb2(), bb3()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4100, "id":4, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4101, "id":5, "opcode":"Return v4", "attributes":[], "inputs":[4], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4099, "id":3, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4098, "id":2, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4099, "id":3, "opcode":"Return v2", "attributes":[], "inputs":[2], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_loop_structure() { + let mut function = Function::new(std::ptr::null()); + + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + let bb3 = function.new_block(0); + + function.push_insn(bb0, Insn::Jump(edge(bb2))); + + let val = function.push_insn(bb2, Insn::Const { val: Const::Value(Qfalse) }); + let _ = function.push_insn(bb2, Insn::CondBranch { val, if_true: edge(bb1), if_false: edge(bb3) }); + let retval = function.push_insn(bb3, Insn::Const { val: Const::CBool(true) }); + let _ = function.push_insn(bb3, Insn::Return { val: retval }); + + function.push_insn(bb1, Insn::Jump(edge(bb2))); + + function.seal_entries(); + let json = function.to_iongraph_pass("loop_structure"); + assert_snapshot!(json.to_string(), @r#"{"name":"loop_structure", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4102, "id":6, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[3], "instructions":[{"ptr":4096, "id":0, "opcode":"Jump bb3()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4099, "id":3, "loopDepth":1, "attributes":["loopheader"], "predecessors":[1, 2], "successors":[2, 4], "instructions":[{"ptr":4097, "id":1, "opcode":"Const Value(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4098, "id":2, "opcode":"CondBranch v1, bb2(), bb4()", "attributes":[], "inputs":[1], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":1, "attributes":["backedge"], "predecessors":[3], "successors":[3], "instructions":[{"ptr":4101, "id":5, "opcode":"Jump bb3()", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4100, "id":4, "loopDepth":0, "attributes":[], "predecessors":[3], "successors":[], "instructions":[{"ptr":4099, "id":3, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4100, "id":4, "opcode":"Return v3", "attributes":[], "inputs":[3], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + + #[test] + fn test_multiple_successors() { + let mut function = Function::new(std::ptr::null()); + let bb0 = function.entry_block; + let bb1 = function.new_block(0); + let bb2 = function.new_block(0); + + let cond = function.push_insn(bb0, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb0, Insn::CondBranch { val: cond, if_true: edge(bb1), if_false: edge(bb2) }); + + let retval1 = function.push_insn(bb1, Insn::Const { val: Const::CBool(true) }); + function.push_insn(bb1, Insn::Return { val: retval1 }); + + let retval2 = function.push_insn(bb2, Insn::Const { val: Const::CBool(false) }); + function.push_insn(bb2, Insn::Return { val: retval2 }); + + function.seal_entries(); + let json = function.to_iongraph_pass("multiple_successors"); + assert_snapshot!(json.to_string(), @r#"{"name":"multiple_successors", "mir":{"blocks":[{"ptr":4096, "id":0, "loopDepth":0, "attributes":[], "predecessors":[], "successors":[1], "instructions":[{"ptr":4102, "id":6, "opcode":"Entries bb1", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4097, "id":1, "loopDepth":0, "attributes":[], "predecessors":[0], "successors":[2, 3], "instructions":[{"ptr":4096, "id":0, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4097, "id":1, "opcode":"CondBranch v0, bb2(), bb3()", "attributes":[], "inputs":[0], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4098, "id":2, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4098, "id":2, "opcode":"Const CBool(true)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4099, "id":3, "opcode":"Return v2", "attributes":[], "inputs":[2], "uses":[], "memInputs":[], "type":""}]}, {"ptr":4099, "id":3, "loopDepth":0, "attributes":[], "predecessors":[1], "successors":[], "instructions":[{"ptr":4100, "id":4, "opcode":"Const CBool(false)", "attributes":[], "inputs":[], "uses":[], "memInputs":[], "type":"Any"}, {"ptr":4101, "id":5, "opcode":"Return v4", "attributes":[], "inputs":[4], "uses":[], "memInputs":[], "type":""}]}]}, "lir":{"blocks":[]}}"#); + } + } diff --git a/zjit/src/hir_effect/gen_hir_effect.rb b/zjit/src/hir_effect/gen_hir_effect.rb new file mode 100644 index 0000000000..1cc552d1e6 --- /dev/null +++ b/zjit/src/hir_effect/gen_hir_effect.rb @@ -0,0 +1,126 @@ +# Generate hir_effect.inc.rs. To do this, we build up a DAG that +# represents the ZJIT effect hierarchy. + +require 'set' + +# Effect represents not just a Ruby class but a named union of other effects. +class Effect + attr_accessor :name, :subeffects + + def initialize name, subeffects=nil + @name = name + @subeffects = subeffects || [] + end + + def all_subeffects + subeffects.flat_map { |subeffect| subeffect.all_subeffects } + subeffects + end + + def subeffect name + result = Effect.new name + @subeffects << result + result + end +end + +# Helper to generate graphviz. +def to_graphviz_rec effect, f + effect.subeffects.each {|subeffect| + f.puts effect.name + "->" + subeffect.name + ";" + } + effect.subeffects.each {|subeffect| + to_graphviz_rec subeffect, f + } +end + +# Generate graphviz. +def to_graphviz effect, f + f.puts "digraph G {" + to_graphviz_rec effect, f + f.puts "}" +end + +# ===== Start generating the effect DAG ===== + +# Start at Any. All effects are subeffects of Any. +any = Effect.new 'Any' +# Build the effect universe. +allocator = any.subeffect 'Allocator' +control = any.subeffect 'Control' +memory = any.subeffect 'Memory' +patch_point = any.subeffect 'PatchPoint' +interrupt_flag = memory.subeffect 'InterruptFlag' +other = memory.subeffect 'Other' +frame = memory.subeffect 'Frame' +pc = frame.subeffect 'PC' +locals = frame.subeffect 'Locals' +stack = frame.subeffect 'Stack' + +# Use the smallest unsigned value needed to describe all effect bits +# If it becomes an issue, this can be generated but for now we do it manually +$int_label = 'u8' + +# Assign individual bits to effect leaves and union bit patterns to nodes with subeffects +num_bits = 0 +$bits = {"Empty" => ["0#{$int_label}"]} +$numeric_bits = {"Empty" => 0} +Set[any, *any.all_subeffects].sort_by(&:name).each {|effect| + subeffects = effect.subeffects + if subeffects.empty? + # Assign bits for leaves + $bits[effect.name] = ["1#{$int_label} << #{num_bits}"] + $numeric_bits[effect.name] = 1 << num_bits + num_bits += 1 + else + # Assign bits for unions + $bits[effect.name] = subeffects.map(&:name).sort + end +} +[*any.all_subeffects, any].each {|effect| + subeffects = effect.subeffects + unless subeffects.empty? + $numeric_bits[effect.name] = subeffects.map {|ty| $numeric_bits[ty.name]}.reduce(&:|) + end +} + +# ===== Finished generating the DAG; write Rust code ===== + +puts "// This file is @generated by src/hir/gen_hir_effect.rb." +puts "mod bits {" +$bits.keys.sort.map {|effect_name| + subeffects = $bits[effect_name].join(" | ") + puts " pub const #{effect_name}: #{$int_label} = #{subeffects};" +} +puts " pub const AllBitPatterns: [(&str, #{$int_label}); #{$bits.size}] = [" +# Sort the bit patterns by decreasing value so that we can print the densest +# possible to-string representation of an Effect. For example, Frame instead of +# PC|Stack|Locals +$numeric_bits.sort_by {|key, val| -val}.each {|effect_name, _| + puts " (\"#{effect_name}\", #{effect_name})," +} +puts " ];" +puts " pub const NumEffectBits: #{$int_label} = #{num_bits}; +}" + +puts "pub mod effect_types {" +puts " pub type EffectBits = #{$int_label};" +puts "}" + +puts "pub mod abstract_heaps { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: AbstractHeap = AbstractHeap::from_bits(bits::#{effect_name});" +} +puts "}" + +puts "pub mod effects { + use super::*;" +$bits.keys.sort.map {|effect_name| + puts " pub const #{effect_name}: Effect = Effect::promote(abstract_heaps::#{effect_name});" +} +puts "}" + +File.open("zjit_effects.dot", "w") do |f| + to_graphviz(any, f) +end + diff --git a/zjit/src/hir_effect/hir_effect.inc.rs b/zjit/src/hir_effect/hir_effect.inc.rs new file mode 100644 index 0000000000..75e3447da1 --- /dev/null +++ b/zjit/src/hir_effect/hir_effect.inc.rs @@ -0,0 +1,63 @@ +// This file is @generated by src/hir/gen_hir_effect.rb. +mod bits { + pub const Allocator: u8 = 1u8 << 0; + pub const Any: u8 = Allocator | Control | Memory | PatchPoint; + pub const Control: u8 = 1u8 << 1; + pub const Empty: u8 = 0u8; + pub const Frame: u8 = Locals | PC | Stack; + pub const InterruptFlag: u8 = 1u8 << 2; + pub const Locals: u8 = 1u8 << 3; + pub const Memory: u8 = Frame | InterruptFlag | Other; + pub const Other: u8 = 1u8 << 4; + pub const PC: u8 = 1u8 << 5; + pub const PatchPoint: u8 = 1u8 << 6; + pub const Stack: u8 = 1u8 << 7; + pub const AllBitPatterns: [(&str, u8); 12] = [ + ("Any", Any), + ("Memory", Memory), + ("Frame", Frame), + ("Stack", Stack), + ("PatchPoint", PatchPoint), + ("PC", PC), + ("Other", Other), + ("Locals", Locals), + ("InterruptFlag", InterruptFlag), + ("Control", Control), + ("Allocator", Allocator), + ("Empty", Empty), + ]; + pub const NumEffectBits: u8 = 8; +} +pub mod effect_types { + pub type EffectBits = u8; +} +pub mod abstract_heaps { + use super::*; + pub const Allocator: AbstractHeap = AbstractHeap::from_bits(bits::Allocator); + pub const Any: AbstractHeap = AbstractHeap::from_bits(bits::Any); + pub const Control: AbstractHeap = AbstractHeap::from_bits(bits::Control); + pub const Empty: AbstractHeap = AbstractHeap::from_bits(bits::Empty); + pub const Frame: AbstractHeap = AbstractHeap::from_bits(bits::Frame); + pub const InterruptFlag: AbstractHeap = AbstractHeap::from_bits(bits::InterruptFlag); + pub const Locals: AbstractHeap = AbstractHeap::from_bits(bits::Locals); + pub const Memory: AbstractHeap = AbstractHeap::from_bits(bits::Memory); + pub const Other: AbstractHeap = AbstractHeap::from_bits(bits::Other); + pub const PC: AbstractHeap = AbstractHeap::from_bits(bits::PC); + pub const PatchPoint: AbstractHeap = AbstractHeap::from_bits(bits::PatchPoint); + pub const Stack: AbstractHeap = AbstractHeap::from_bits(bits::Stack); +} +pub mod effects { + use super::*; + pub const Allocator: Effect = Effect::promote(abstract_heaps::Allocator); + pub const Any: Effect = Effect::promote(abstract_heaps::Any); + pub const Control: Effect = Effect::promote(abstract_heaps::Control); + pub const Empty: Effect = Effect::promote(abstract_heaps::Empty); + pub const Frame: Effect = Effect::promote(abstract_heaps::Frame); + pub const InterruptFlag: Effect = Effect::promote(abstract_heaps::InterruptFlag); + pub const Locals: Effect = Effect::promote(abstract_heaps::Locals); + pub const Memory: Effect = Effect::promote(abstract_heaps::Memory); + pub const Other: Effect = Effect::promote(abstract_heaps::Other); + pub const PC: Effect = Effect::promote(abstract_heaps::PC); + pub const PatchPoint: Effect = Effect::promote(abstract_heaps::PatchPoint); + pub const Stack: Effect = Effect::promote(abstract_heaps::Stack); +} diff --git a/zjit/src/hir_effect/mod.rs b/zjit/src/hir_effect/mod.rs new file mode 100644 index 0000000000..b1d7b27411 --- /dev/null +++ b/zjit/src/hir_effect/mod.rs @@ -0,0 +1,420 @@ +//! High-level intermediate representation effects. + +#![allow(non_upper_case_globals)] +use crate::hir::{PtrPrintMap}; +include!("hir_effect.inc.rs"); + +// NOTE: Effect very intentionally does not support Eq or PartialEq; we almost never want to check +// bit equality of types in the compiler but instead check subtyping, intersection, union, etc. +/// The AbstractHeap struct is the main work horse of effect inference and specialization. The main interfaces +/// will look like: +/// +/// * is AbstractHeap A a subset of AbstractHeap B +/// * union/meet AbstractHeap A and AbstractHeap B +/// +/// or +/// +/// * is Effect A a subset of Effect B +/// * union/meet Effect A and Effect B +/// +/// The AbstractHeap is the work horse because Effect is simply 2 AbstractHeaps; one for read, and one for write. +/// Currently, the abstract heap is implemented as a bitset. As we enrich our effect system, this will be updated +/// to match the name and use a heap implementation, roughly aligned with +/// <https://gist.github.com/pizlonator/cf1e72b8600b1437dda8153ea3fdb963>. +/// +/// Most questions can be rewritten in terms of these operations. +/// +/// Lattice Top corresponds to the "Any" effect. All bits are set and any effect is possible. +/// Lattice Bottom corresponds to the "None" effect. No bits are set and no effects are possible. +/// Elements between abstract_heaps have effects corresponding to the bits that are set. +/// This enables more complex analyses compared to prior ZJIT implementations such as "has_effect", +/// a function that returns a boolean value. Such functions impose an implicit single bit effect +/// system. This explicit design with a lattice allows us many bits for effects. +#[derive(Clone, Copy, Debug)] +pub struct AbstractHeap { + bits: effect_types::EffectBits +} + +#[derive(Clone, Copy, Debug)] +pub struct Effect { + /// Unlike ZJIT's type system, effects do not have a notion of subclasses. + /// Instead of specializations, the Effect struct contains two AbstractHeaps. + /// We distinguish between read effects and write effects. + /// Both use the same effects lattice, but splitting into two heaps allows + /// for finer grained optimization. + /// + /// For instance: + /// We can elide HIR instructions with no write effects, but the read effects are necessary for instruction + /// reordering optimizations. + /// + /// These fields should not be directly read or written except by internal `Effect` APIs. + read: AbstractHeap, + write: AbstractHeap +} + +/// Print adaptor for [`AbstractHeap`]. See [`PtrPrintMap`]. +pub struct AbstractHeapPrinter<'a> { + inner: AbstractHeap, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for AbstractHeapPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let effect = self.inner; + let mut bits = effect.bits; + let mut sep = ""; + // First, make sure patterns are sorted from higher order bits to lower order. + // For each match where `bits` contains the pattern, we mask off the matched bits + // and continue searching for matches until bits == 0. + // Our first match could be exact and may not require a separator, but all subsequent + // matches do. + debug_assert!(bits::AllBitPatterns.is_sorted_by(|(_, left), (_, right)| left > right)); + for (name, pattern) in bits::AllBitPatterns { + if (bits & pattern) == pattern { + write!(f, "{sep}{name}")?; + sep = "|"; + bits &= !pattern; + } + // The `sep != ""` check allows us to handle the effects::None case gracefully. + if (bits == 0) & (sep != "") { break; } + } + debug_assert_eq!(bits, 0, "Should have eliminated all bits by iterating over all patterns"); + Ok(()) + } +} + +impl std::fmt::Display for AbstractHeap { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +/// Print adaptor for [`Effect`]. See [`PtrPrintMap`]. +pub struct EffectPrinter<'a> { + inner: Effect, + ptr_map: &'a PtrPrintMap, +} + +impl<'a> std::fmt::Display for EffectPrinter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}, {}", self.inner.read, self.inner.write) + } +} + +impl std::fmt::Display for Effect { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.print(&PtrPrintMap::identity()).fmt(f) + } +} + +impl AbstractHeap { + const fn from_bits(bits: effect_types::EffectBits) -> Self { + Self { bits } + } + + pub const fn union(&self, other: Self) -> Self { + Self::from_bits(self.bits | other.bits) + } + + pub const fn intersect(&self, other: Self) -> Self { + Self::from_bits(self.bits & other.bits) + } + + pub const fn exclude(&self, other: Self) -> Self { + Self::from_bits(self.bits - (self.bits & other.bits)) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Self) -> bool { + self.bits == other.bits + } + + pub const fn includes(&self, other: Self) -> bool { + self.bit_equal( + self.union(other) + ) + } + + pub const fn overlaps(&self, other: Self) -> bool { + !abstract_heaps::Empty.includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> AbstractHeapPrinter<'_> { + AbstractHeapPrinter { inner: self, ptr_map } + } +} + +impl Effect { + pub const fn read_write(read: AbstractHeap, write: AbstractHeap) -> Effect { + Effect { read, write } + } + + /// This function addresses the special case where the read and write heaps are the same + pub const fn promote(heap: AbstractHeap) -> Effect { + Effect {read: heap, write: heap } + } + + /// This function accepts write and heaps read to Any + pub const fn write(write: AbstractHeap) -> Effect { + Effect { read: abstract_heaps::Any, write } + } + + /// This function accepts read and heaps read to Any + pub const fn read(read: AbstractHeap) -> Effect { + Effect { read, write: abstract_heaps::Any } + } + + /// Method to access the private read field + pub const fn read_bits(&self) -> AbstractHeap { + self.read + } + + /// Method to access the private write field + pub const fn write_bits(&self) -> AbstractHeap { + self.write + } + + pub const fn union(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.union(other.read), + self.write.union(other.write) + ) + } + + pub const fn intersect(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.intersect(other.read), + self.write.intersect(other.write) + ) + } + + pub const fn exclude(&self, other: Effect) -> Effect { + Effect::read_write( + self.read.exclude(other.read), + self.write.exclude(other.write) + ) + } + + /// Check bit equality of two `Effect`s. Do not use! You are probably looking for [`Effect::includes`]. + /// This function is intentionally made private. + const fn bit_equal(&self, other: Effect) -> bool { + self.read.bit_equal(other.read) & self.write.bit_equal(other.write) + } + + pub const fn includes(&self, other: Effect) -> bool { + self.bit_equal(Effect::union(self, other)) + } + + pub const fn overlaps(&self, other: Effect) -> bool { + Effect::promote(abstract_heaps::Empty).includes( + self.intersect(other) + ) + } + + pub const fn print(self, ptr_map: &PtrPrintMap) -> EffectPrinter<'_> { + EffectPrinter { inner: self, ptr_map } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[track_caller] + fn assert_heap_bit_equal(left: AbstractHeap, right: AbstractHeap) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(right.includes(left), "{left} is not a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_not_subeffect_heap(left: AbstractHeap, right: AbstractHeap) { + assert!(!right.includes(left), "{left} is a subeffect heap of {right}"); + } + + #[track_caller] + fn assert_bit_equal(left: Effect, right: Effect) { + assert!(left.bit_equal(right), "{left} bits are not equal to {right} bits"); + } + + #[track_caller] + fn assert_subeffect(left: Effect, right: Effect) { + assert!(right.includes(left), "{left} is not a subeffect of {right}"); + } + + #[track_caller] + fn assert_not_subeffect(left: Effect, right: Effect) { + assert!(!right.includes(left), "{left} is a subeffect of {right}"); + } + + #[test] + fn effect_heap_none_is_subeffect_of_everything() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Control); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Stack); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Locals); + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Allocator); + } + + #[test] + fn effect_heap_everything_is_subeffect_of_any() { + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Control, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Any); + } + + #[test] + fn effect_heap_union_never_shrinks() { + // iterate over all effect entries from bottom to top + for i in [0, 1, 4, 6, 10, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty, abstract_heaps::Empty.union(e)); + assert_subeffect_heap(abstract_heaps::Any, abstract_heaps::Any.union(e)); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Frame.union(e)); + } + } + + #[test] + fn effect_heap_intersect_never_grows() { + // Randomly selected values from bottom to top + for i in [0, 3, 6, 8, 15] { + let e = AbstractHeap::from_bits(i); + // Testing on bottom, top, and some arbitrary element in the middle + assert_subeffect_heap(abstract_heaps::Empty.intersect(e), abstract_heaps::Empty); + assert_subeffect_heap(abstract_heaps::Any.intersect(e), abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame.intersect(e), abstract_heaps::Frame); + } + } + + #[test] + fn effect_heap_self_is_included() { + assert!(abstract_heaps::Stack.includes(abstract_heaps::Stack)); + assert!(abstract_heaps::Any.includes(abstract_heaps::Any)); + assert!(abstract_heaps::Empty.includes(abstract_heaps::Empty)); + } + + #[test] + fn effect_heap_frame_includes_stack_locals_and_pc() { + assert_subeffect_heap(abstract_heaps::Stack, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::Locals, abstract_heaps::Frame); + assert_subeffect_heap(abstract_heaps::PC, abstract_heaps::Frame); + } + + #[test] + fn effect_heap_frame_is_stack_locals_and_pc() { + let union = abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC)); + assert_heap_bit_equal(abstract_heaps::Frame, union); + } + + #[test] + fn effect_heap_any_includes_some_subeffects() { + assert_subeffect_heap(abstract_heaps::Allocator, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Frame, abstract_heaps::Any); + assert_subeffect_heap(abstract_heaps::Memory, abstract_heaps::Any); + } + + #[test] + fn effect_heap_display_exact_bits_match() { + assert_eq!(format!("{}", abstract_heaps::Empty), "Empty"); + assert_eq!(format!("{}", abstract_heaps::PC), "PC"); + assert_eq!(format!("{}", abstract_heaps::Any), "Any"); + assert_eq!(format!("{}", abstract_heaps::Frame), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + } + + #[test] + fn effect_heap_display_multiple_bits() { + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals.union(abstract_heaps::PC))), "Frame"); + assert_eq!(format!("{}", abstract_heaps::Stack.union(abstract_heaps::Locals)), "Stack|Locals"); + assert_eq!(format!("{}", abstract_heaps::PC.union(abstract_heaps::Allocator)), "PC|Allocator"); + } + + #[test] + fn effect_any_includes_everything() { + assert_subeffect(effects::Allocator, effects::Any); + assert_subeffect(effects::Frame, effects::Any); + assert_subeffect(effects::Memory, effects::Any); + // Let's do a less standard effect too + assert_subeffect( + Effect::read_write(abstract_heaps::Control, abstract_heaps::Any), + effects::Any + ); + } + + #[test] + fn effect_union_is_idempotent() { + assert_bit_equal( + Effect::read(abstract_heaps::Any) + .union(Effect::write(abstract_heaps::Any)), + effects::Any + ); + assert_bit_equal( + effects::Empty.union(effects::Empty), + effects::Empty + ); + } + + #[test] + fn effect_union_contains_and_excludes() { + assert_subeffect( + effects::Control.union(effects::Frame), + effects::Any + ); + assert_not_subeffect( + effects::Frame.union(effects::Locals), + effects::PC + ); + } + + #[test] + fn effect_intersect_is_empty() { + assert_subeffect(effects::Memory.intersect(effects::Control), effects::Empty); + assert_subeffect( + Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Other) + .intersect(Effect::read_write(abstract_heaps::Stack, abstract_heaps::PC)), + effects::Empty + ) + } + + #[test] + fn effect_intersect_exact_match() { + assert_subeffect(effects::Frame.intersect(effects::PC), effects::PC); + assert_subeffect(effects::Allocator.intersect(effects::Allocator), effects::Allocator); + } + + #[test] + fn effect_display_exact_bits_match() { + assert_eq!(format!("{}", effects::Empty), "Empty, Empty"); + assert_eq!(format!("{}", effects::PC), "PC, PC"); + assert_eq!(format!("{}", effects::Any), "Any, Any"); + assert_eq!(format!("{}", effects::Frame), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", Effect::write(abstract_heaps::Control)), "Any, Control"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Allocator, abstract_heaps::Memory)), "Allocator, Memory"); + } + + #[test] + fn effect_display_multiple_bits() { + assert_eq!(format!("{}", effects::Stack.union(effects::Locals.union(effects::PC))), "Frame, Frame"); + assert_eq!(format!("{}", effects::Stack.union(effects::Locals)), "Stack|Locals, Stack|Locals"); + assert_eq!(format!("{}", effects::PC.union(effects::Allocator)), "PC|Allocator, PC|Allocator"); + assert_eq!(format!("{}", Effect::read_write(abstract_heaps::Other, abstract_heaps::PC) + .union(Effect::read_write(abstract_heaps::Memory, abstract_heaps::Stack))), + "Memory, Stack|PC" + ); + } + +} diff --git a/zjit/src/hir_type/gen_hir_type.rb b/zjit/src/hir_type/gen_hir_type.rb index 15aa68a600..2eb5ca1932 100644 --- a/zjit/src/hir_type/gen_hir_type.rb +++ b/zjit/src/hir_type/gen_hir_type.rb @@ -25,20 +25,20 @@ class Type end # Helper to generate graphviz. -def to_graphviz_rec type +def to_graphviz_rec type, f type.subtypes.each {|subtype| - puts type.name + "->" + subtype.name + ";" + f.puts type.name + "->" + subtype.name + ";" } type.subtypes.each {|subtype| - to_graphviz_rec subtype + to_graphviz_rec subtype, f } end # Generate graphviz. -def to_graphviz type - puts "digraph G {" - to_graphviz_rec type - puts "}" +def to_graphviz type, f + f.puts "digraph G {" + to_graphviz_rec type, f + f.puts "}" end # ===== Start generating the type DAG ===== @@ -58,50 +58,85 @@ object_subclass = $object.subtype "ObjectSubclass" $subclass = [basic_object_subclass.name, object_subclass.name] $builtin_exact = [basic_object_exact.name, object_exact.name] +$exact_c_names = { + "ObjectExact" => "rb_cObject", + "BasicObjectExact" => "rb_cBasicObject", +} + +$subclass_c_names = { + "ObjectSubclass" => "rb_cObject", + "BasicObjectSubclass" => "rb_cBasicObject", +} + +$inexact_c_names = { + "Object" => "rb_cObject", + "BasicObject" => "rb_cBasicObject", +} + # Define a new type that can be subclassed (most of them). -def base_type name - type = $object.subtype name +# If c_name is given, mark the rb_cXYZ object as equivalent to this exact type. +def base_type name, base: $object, c_name: nil + type = base.subtype name exact = type.subtype(name+"Exact") subclass = type.subtype(name+"Subclass") + if c_name + $exact_c_names[exact.name] = c_name + $subclass_c_names[subclass.name] = c_name + $inexact_c_names[type.name] = c_name + end $builtin_exact << exact.name $subclass << subclass.name [type, exact] end -# Define a new type that cannot be subclassed. -def final_type name - type = $object.subtype name +# Define a new type that has no subclasses and cannot be subclassed. +# If c_name is given, mark the rb_cXYZ object as equivalent to this type. +def final_type name, base: $object, c_name: nil + if c_name + $exact_c_names[name] = c_name + $subclass_c_names[name] = c_name + $inexact_c_names[name] = c_name + end + type = base.subtype name $builtin_exact << type.name type end -base_type "String" -base_type "Array" -base_type "Hash" -base_type "Range" -base_type "Set" -base_type "Regexp" -module_class, _ = base_type "Module" -module_class.subtype "Class" +base_type "String", c_name: "rb_cString" +base_type "Array", c_name: "rb_cArray" +base_type "Hash", c_name: "rb_cHash" +base_type "Range", c_name: "rb_cRange" +base_type "Set", c_name: "rb_cSet" +base_type "Regexp", c_name: "rb_cRegexp" +module_class, _ = base_type "Module", c_name: "rb_cModule" +# Class cannot be subclassed by doing `class Sub < Class`, +# but every metaclass is a subclass of `Class`. It's not final. +base_type "Class", base: module_class, c_name: "rb_cClass" + +numeric, _ = base_type "Numeric", c_name: "rb_cNumeric" -integer_exact = final_type "Integer" +integer_exact = final_type "Integer", base: numeric, c_name: "rb_cInteger" # CRuby partitions Integer into immediate and non-immediate variants. fixnum = integer_exact.subtype "Fixnum" integer_exact.subtype "Bignum" -float_exact = final_type "Float" +float_exact = final_type "Float", base: numeric, c_name: "rb_cFloat" # CRuby partitions Float into immediate and non-immediate variants. flonum = float_exact.subtype "Flonum" float_exact.subtype "HeapFloat" -symbol_exact = final_type "Symbol" +symbol_exact = final_type "Symbol", c_name: "rb_cSymbol" # CRuby partitions Symbol into immediate and non-immediate variants. static_sym = symbol_exact.subtype "StaticSymbol" symbol_exact.subtype "DynamicSymbol" -nil_exact = final_type "NilClass" -true_exact = final_type "TrueClass" -false_exact = final_type "FalseClass" +nil_exact = final_type "NilClass", c_name: "rb_cNilClass" +true_exact = final_type "TrueClass", c_name: "rb_cTrueClass" +false_exact = final_type "FalseClass", c_name: "rb_cFalseClass" + +# T_DATA objects have a distinct memory layout for field access and don't have a +# common class ancestor below BasicObject. +basic_object.subtype "TData" # Build the cvalue object universe. This is for C-level types that may be # passed around when calling into the Ruby VM or after some strength reduction @@ -118,6 +153,8 @@ unsigned = cvalue_int.subtype "CUnsigned" signed.subtype "CInt#{width}" unsigned.subtype "CUInt#{width}" } +unsigned.subtype "CShape" +unsigned.subtype "CAttrIndex" # Assign individual bits to type leaves and union bit patterns to nodes with subtypes num_bits = 0 @@ -156,8 +193,15 @@ add_union "BuiltinExact", $builtin_exact add_union "Subclass", $subclass add_union "BoolExact", [true_exact.name, false_exact.name] add_union "Immediate", [fixnum.name, flonum.name, static_sym.name, nil_exact.name, true_exact.name, false_exact.name, undef_.name] -$bits["HeapObject"] = ["BasicObject & !Immediate"] -$numeric_bits["HeapObject"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Immediate"] +add_union "Falsy", [nil_exact.name, false_exact.name] +$bits["HeapBasicObject"] = ["BasicObject & !Immediate"] +$numeric_bits["HeapBasicObject"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Immediate"] +$bits["HeapObject"] = ["Object & !Immediate"] +$numeric_bits["HeapObject"] = $numeric_bits["Object"] & ~$numeric_bits["Immediate"] +$bits["Truthy"] = ["BasicObject & !Falsy"] +$numeric_bits["Truthy"] = $numeric_bits["BasicObject"] & ~$numeric_bits["Falsy"] +$bits["NotNil"] = ["BasicObject & !NilClass"] +$numeric_bits["NotNil"] = $numeric_bits["BasicObject"] & ~$numeric_bits["NilClass"] # ===== Finished generating the DAG; write Rust code ===== @@ -167,7 +211,7 @@ $bits.keys.sort.map {|type_name| subtypes = $bits[type_name].join(" | ") puts " pub const #{type_name}: u64 = #{subtypes};" } -puts " pub const AllBitPatterns: [(&'static str, u64); #{$bits.size}] = [" +puts " pub const AllBitPatterns: [(&str, u64); #{$bits.size}] = [" # Sort the bit patterns by decreasing value so that we can print the densest # possible to-string representation of a Type. For example, CSigned instead of # CInt8|CInt16|... @@ -183,4 +227,25 @@ puts "pub mod types { $bits.keys.sort.map {|type_name| puts " pub const #{type_name}: Type = Type::from_bits(bits::#{type_name});" } +puts " pub const ExactBitsAndClass: [(u64, *const VALUE); #{$exact_c_names.size}] = [" +$exact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +$subclass_c_names = $subclass_c_names.to_a.sort_by {|name, _| $numeric_bits[name.sub("Subclass", "")]}.to_h +puts " pub const SubclassBitsAndClass: [(u64, *const VALUE); #{$subclass_c_names.size}] = [" +$subclass_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" +$inexact_c_names = $inexact_c_names.to_a.sort_by {|name, _| $numeric_bits[name]}.to_h +puts " pub const InexactBitsAndClass: [(u64, *const VALUE); #{$inexact_c_names.size}] = [" +$inexact_c_names.each {|type_name, c_name| + puts " (bits::#{type_name}, &raw const crate::cruby::#{c_name})," +} +puts " ];" puts "}" + +File.open("zjit_types.dot", "w") do |f| + to_graphviz(any, f) +end diff --git a/zjit/src/hir_type/hir_type.inc.rs b/zjit/src/hir_type/hir_type.inc.rs index 5850874080..1dae344b36 100644 --- a/zjit/src/hir_type/hir_type.inc.rs +++ b/zjit/src/hir_type/hir_type.inc.rs @@ -4,74 +4,90 @@ mod bits { pub const Array: u64 = ArrayExact | ArraySubclass; pub const ArrayExact: u64 = 1u64 << 0; pub const ArraySubclass: u64 = 1u64 << 1; - pub const BasicObject: u64 = BasicObjectExact | BasicObjectSubclass | Object; + pub const BasicObject: u64 = BasicObjectExact | BasicObjectSubclass | Object | TData; pub const BasicObjectExact: u64 = 1u64 << 2; pub const BasicObjectSubclass: u64 = 1u64 << 3; pub const Bignum: u64 = 1u64 << 4; pub const BoolExact: u64 = FalseClass | TrueClass; - pub const BuiltinExact: u64 = ArrayExact | BasicObjectExact | FalseClass | Float | HashExact | Integer | ModuleExact | NilClass | ObjectExact | RangeExact | RegexpExact | SetExact | StringExact | Symbol | TrueClass; - pub const CBool: u64 = 1u64 << 5; - pub const CDouble: u64 = 1u64 << 6; + pub const BuiltinExact: u64 = ArrayExact | BasicObjectExact | ClassExact | FalseClass | Float | HashExact | Integer | ModuleExact | NilClass | NumericExact | ObjectExact | RangeExact | RegexpExact | SetExact | StringExact | Symbol | TrueClass; + pub const CAttrIndex: u64 = 1u64 << 5; + pub const CBool: u64 = 1u64 << 6; + pub const CDouble: u64 = 1u64 << 7; pub const CInt: u64 = CSigned | CUnsigned; - pub const CInt16: u64 = 1u64 << 7; - pub const CInt32: u64 = 1u64 << 8; - pub const CInt64: u64 = 1u64 << 9; - pub const CInt8: u64 = 1u64 << 10; - pub const CNull: u64 = 1u64 << 11; - pub const CPtr: u64 = 1u64 << 12; + pub const CInt16: u64 = 1u64 << 8; + pub const CInt32: u64 = 1u64 << 9; + pub const CInt64: u64 = 1u64 << 10; + pub const CInt8: u64 = 1u64 << 11; + pub const CNull: u64 = 1u64 << 12; + pub const CPtr: u64 = 1u64 << 13; + pub const CShape: u64 = 1u64 << 14; pub const CSigned: u64 = CInt16 | CInt32 | CInt64 | CInt8; - pub const CUInt16: u64 = 1u64 << 13; - pub const CUInt32: u64 = 1u64 << 14; - pub const CUInt64: u64 = 1u64 << 15; - pub const CUInt8: u64 = 1u64 << 16; - pub const CUnsigned: u64 = CUInt16 | CUInt32 | CUInt64 | CUInt8; + pub const CUInt16: u64 = 1u64 << 15; + pub const CUInt32: u64 = 1u64 << 16; + pub const CUInt64: u64 = 1u64 << 17; + pub const CUInt8: u64 = 1u64 << 18; + pub const CUnsigned: u64 = CAttrIndex | CShape | CUInt16 | CUInt32 | CUInt64 | CUInt8; pub const CValue: u64 = CBool | CDouble | CInt | CNull | CPtr; - pub const CallableMethodEntry: u64 = 1u64 << 17; - pub const Class: u64 = 1u64 << 18; - pub const DynamicSymbol: u64 = 1u64 << 19; + pub const CallableMethodEntry: u64 = 1u64 << 19; + pub const Class: u64 = ClassExact | ClassSubclass; + pub const ClassExact: u64 = 1u64 << 20; + pub const ClassSubclass: u64 = 1u64 << 21; + pub const DynamicSymbol: u64 = 1u64 << 22; pub const Empty: u64 = 0u64; - pub const FalseClass: u64 = 1u64 << 20; - pub const Fixnum: u64 = 1u64 << 21; + pub const FalseClass: u64 = 1u64 << 23; + pub const Falsy: u64 = FalseClass | NilClass; + pub const Fixnum: u64 = 1u64 << 24; pub const Float: u64 = Flonum | HeapFloat; - pub const Flonum: u64 = 1u64 << 22; + pub const Flonum: u64 = 1u64 << 25; pub const Hash: u64 = HashExact | HashSubclass; - pub const HashExact: u64 = 1u64 << 23; - pub const HashSubclass: u64 = 1u64 << 24; - pub const HeapFloat: u64 = 1u64 << 25; - pub const HeapObject: u64 = BasicObject & !Immediate; + pub const HashExact: u64 = 1u64 << 26; + pub const HashSubclass: u64 = 1u64 << 27; + pub const HeapBasicObject: u64 = BasicObject & !Immediate; + pub const HeapFloat: u64 = 1u64 << 28; + pub const HeapObject: u64 = Object & !Immediate; pub const Immediate: u64 = FalseClass | Fixnum | Flonum | NilClass | StaticSymbol | TrueClass | Undef; pub const Integer: u64 = Bignum | Fixnum; pub const Module: u64 = Class | ModuleExact | ModuleSubclass; - pub const ModuleExact: u64 = 1u64 << 26; - pub const ModuleSubclass: u64 = 1u64 << 27; - pub const NilClass: u64 = 1u64 << 28; - pub const Object: u64 = Array | FalseClass | Float | Hash | Integer | Module | NilClass | ObjectExact | ObjectSubclass | Range | Regexp | Set | String | Symbol | TrueClass; - pub const ObjectExact: u64 = 1u64 << 29; - pub const ObjectSubclass: u64 = 1u64 << 30; + pub const ModuleExact: u64 = 1u64 << 29; + pub const ModuleSubclass: u64 = 1u64 << 30; + pub const NilClass: u64 = 1u64 << 31; + pub const NotNil: u64 = BasicObject & !NilClass; + pub const Numeric: u64 = Float | Integer | NumericExact | NumericSubclass; + pub const NumericExact: u64 = 1u64 << 32; + pub const NumericSubclass: u64 = 1u64 << 33; + pub const Object: u64 = Array | FalseClass | Hash | Module | NilClass | Numeric | ObjectExact | ObjectSubclass | Range | Regexp | Set | String | Symbol | TrueClass; + pub const ObjectExact: u64 = 1u64 << 34; + pub const ObjectSubclass: u64 = 1u64 << 35; pub const Range: u64 = RangeExact | RangeSubclass; - pub const RangeExact: u64 = 1u64 << 31; - pub const RangeSubclass: u64 = 1u64 << 32; + pub const RangeExact: u64 = 1u64 << 36; + pub const RangeSubclass: u64 = 1u64 << 37; pub const Regexp: u64 = RegexpExact | RegexpSubclass; - pub const RegexpExact: u64 = 1u64 << 33; - pub const RegexpSubclass: u64 = 1u64 << 34; + pub const RegexpExact: u64 = 1u64 << 38; + pub const RegexpSubclass: u64 = 1u64 << 39; pub const RubyValue: u64 = BasicObject | CallableMethodEntry | Undef; pub const Set: u64 = SetExact | SetSubclass; - pub const SetExact: u64 = 1u64 << 35; - pub const SetSubclass: u64 = 1u64 << 36; - pub const StaticSymbol: u64 = 1u64 << 37; + pub const SetExact: u64 = 1u64 << 40; + pub const SetSubclass: u64 = 1u64 << 41; + pub const StaticSymbol: u64 = 1u64 << 42; pub const String: u64 = StringExact | StringSubclass; - pub const StringExact: u64 = 1u64 << 38; - pub const StringSubclass: u64 = 1u64 << 39; - pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | HashSubclass | ModuleSubclass | ObjectSubclass | RangeSubclass | RegexpSubclass | SetSubclass | StringSubclass; + pub const StringExact: u64 = 1u64 << 43; + pub const StringSubclass: u64 = 1u64 << 44; + pub const Subclass: u64 = ArraySubclass | BasicObjectSubclass | ClassSubclass | HashSubclass | ModuleSubclass | NumericSubclass | ObjectSubclass | RangeSubclass | RegexpSubclass | SetSubclass | StringSubclass; pub const Symbol: u64 = DynamicSymbol | StaticSymbol; - pub const TrueClass: u64 = 1u64 << 40; - pub const Undef: u64 = 1u64 << 41; - pub const AllBitPatterns: [(&'static str, u64); 66] = [ + pub const TrueClass: u64 = 1u64 << 45; + pub const Truthy: u64 = BasicObject & !Falsy; + pub const TData: u64 = 1u64 << 46; + pub const Undef: u64 = 1u64 << 47; + pub const AllBitPatterns: [(&str, u64); 78] = [ ("Any", Any), ("RubyValue", RubyValue), ("Immediate", Immediate), ("Undef", Undef), ("BasicObject", BasicObject), + ("NotNil", NotNil), + ("Truthy", Truthy), + ("HeapBasicObject", HeapBasicObject), + ("TData", TData), ("Object", Object), ("BuiltinExact", BuiltinExact), ("BoolExact", BoolExact), @@ -94,6 +110,10 @@ mod bits { ("RangeExact", RangeExact), ("ObjectSubclass", ObjectSubclass), ("ObjectExact", ObjectExact), + ("Numeric", Numeric), + ("NumericSubclass", NumericSubclass), + ("NumericExact", NumericExact), + ("Falsy", Falsy), ("NilClass", NilClass), ("Module", Module), ("ModuleSubclass", ModuleSubclass), @@ -109,6 +129,8 @@ mod bits { ("FalseClass", FalseClass), ("DynamicSymbol", DynamicSymbol), ("Class", Class), + ("ClassSubclass", ClassSubclass), + ("ClassExact", ClassExact), ("CallableMethodEntry", CallableMethodEntry), ("CValue", CValue), ("CInt", CInt), @@ -117,6 +139,7 @@ mod bits { ("CUInt64", CUInt64), ("CUInt32", CUInt32), ("CUInt16", CUInt16), + ("CShape", CShape), ("CPtr", CPtr), ("CNull", CNull), ("CSigned", CSigned), @@ -126,6 +149,7 @@ mod bits { ("CInt16", CInt16), ("CDouble", CDouble), ("CBool", CBool), + ("CAttrIndex", CAttrIndex), ("Bignum", Bignum), ("BasicObjectSubclass", BasicObjectSubclass), ("BasicObjectExact", BasicObjectExact), @@ -134,7 +158,7 @@ mod bits { ("ArrayExact", ArrayExact), ("Empty", Empty), ]; - pub const NumTypeBits: u64 = 42; + pub const NumTypeBits: u64 = 48; } pub mod types { use super::*; @@ -148,6 +172,7 @@ pub mod types { pub const Bignum: Type = Type::from_bits(bits::Bignum); pub const BoolExact: Type = Type::from_bits(bits::BoolExact); pub const BuiltinExact: Type = Type::from_bits(bits::BuiltinExact); + pub const CAttrIndex: Type = Type::from_bits(bits::CAttrIndex); pub const CBool: Type = Type::from_bits(bits::CBool); pub const CDouble: Type = Type::from_bits(bits::CDouble); pub const CInt: Type = Type::from_bits(bits::CInt); @@ -157,6 +182,7 @@ pub mod types { pub const CInt8: Type = Type::from_bits(bits::CInt8); pub const CNull: Type = Type::from_bits(bits::CNull); pub const CPtr: Type = Type::from_bits(bits::CPtr); + pub const CShape: Type = Type::from_bits(bits::CShape); pub const CSigned: Type = Type::from_bits(bits::CSigned); pub const CUInt16: Type = Type::from_bits(bits::CUInt16); pub const CUInt32: Type = Type::from_bits(bits::CUInt32); @@ -166,15 +192,19 @@ pub mod types { pub const CValue: Type = Type::from_bits(bits::CValue); pub const CallableMethodEntry: Type = Type::from_bits(bits::CallableMethodEntry); pub const Class: Type = Type::from_bits(bits::Class); + pub const ClassExact: Type = Type::from_bits(bits::ClassExact); + pub const ClassSubclass: Type = Type::from_bits(bits::ClassSubclass); pub const DynamicSymbol: Type = Type::from_bits(bits::DynamicSymbol); pub const Empty: Type = Type::from_bits(bits::Empty); pub const FalseClass: Type = Type::from_bits(bits::FalseClass); + pub const Falsy: Type = Type::from_bits(bits::Falsy); pub const Fixnum: Type = Type::from_bits(bits::Fixnum); pub const Float: Type = Type::from_bits(bits::Float); pub const Flonum: Type = Type::from_bits(bits::Flonum); pub const Hash: Type = Type::from_bits(bits::Hash); pub const HashExact: Type = Type::from_bits(bits::HashExact); pub const HashSubclass: Type = Type::from_bits(bits::HashSubclass); + pub const HeapBasicObject: Type = Type::from_bits(bits::HeapBasicObject); pub const HeapFloat: Type = Type::from_bits(bits::HeapFloat); pub const HeapObject: Type = Type::from_bits(bits::HeapObject); pub const Immediate: Type = Type::from_bits(bits::Immediate); @@ -183,6 +213,10 @@ pub mod types { pub const ModuleExact: Type = Type::from_bits(bits::ModuleExact); pub const ModuleSubclass: Type = Type::from_bits(bits::ModuleSubclass); pub const NilClass: Type = Type::from_bits(bits::NilClass); + pub const NotNil: Type = Type::from_bits(bits::NotNil); + pub const Numeric: Type = Type::from_bits(bits::Numeric); + pub const NumericExact: Type = Type::from_bits(bits::NumericExact); + pub const NumericSubclass: Type = Type::from_bits(bits::NumericSubclass); pub const Object: Type = Type::from_bits(bits::Object); pub const ObjectExact: Type = Type::from_bits(bits::ObjectExact); pub const ObjectSubclass: Type = Type::from_bits(bits::ObjectSubclass); @@ -203,5 +237,64 @@ pub mod types { pub const Subclass: Type = Type::from_bits(bits::Subclass); pub const Symbol: Type = Type::from_bits(bits::Symbol); pub const TrueClass: Type = Type::from_bits(bits::TrueClass); + pub const Truthy: Type = Type::from_bits(bits::Truthy); + pub const TData: Type = Type::from_bits(bits::TData); pub const Undef: Type = Type::from_bits(bits::Undef); + pub const ExactBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::ObjectExact, &raw const crate::cruby::rb_cObject), + (bits::BasicObjectExact, &raw const crate::cruby::rb_cBasicObject), + (bits::StringExact, &raw const crate::cruby::rb_cString), + (bits::ArrayExact, &raw const crate::cruby::rb_cArray), + (bits::HashExact, &raw const crate::cruby::rb_cHash), + (bits::RangeExact, &raw const crate::cruby::rb_cRange), + (bits::SetExact, &raw const crate::cruby::rb_cSet), + (bits::RegexpExact, &raw const crate::cruby::rb_cRegexp), + (bits::ModuleExact, &raw const crate::cruby::rb_cModule), + (bits::ClassExact, &raw const crate::cruby::rb_cClass), + (bits::NumericExact, &raw const crate::cruby::rb_cNumeric), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + ]; + pub const SubclassBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::ArraySubclass, &raw const crate::cruby::rb_cArray), + (bits::ClassSubclass, &raw const crate::cruby::rb_cClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::HashSubclass, &raw const crate::cruby::rb_cHash), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::ModuleSubclass, &raw const crate::cruby::rb_cModule), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::NumericSubclass, &raw const crate::cruby::rb_cNumeric), + (bits::RangeSubclass, &raw const crate::cruby::rb_cRange), + (bits::RegexpSubclass, &raw const crate::cruby::rb_cRegexp), + (bits::SetSubclass, &raw const crate::cruby::rb_cSet), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::StringSubclass, &raw const crate::cruby::rb_cString), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::ObjectSubclass, &raw const crate::cruby::rb_cObject), + (bits::BasicObjectSubclass, &raw const crate::cruby::rb_cBasicObject), + ]; + pub const InexactBitsAndClass: [(u64, *const VALUE); 17] = [ + (bits::Array, &raw const crate::cruby::rb_cArray), + (bits::Class, &raw const crate::cruby::rb_cClass), + (bits::FalseClass, &raw const crate::cruby::rb_cFalseClass), + (bits::Integer, &raw const crate::cruby::rb_cInteger), + (bits::Hash, &raw const crate::cruby::rb_cHash), + (bits::Float, &raw const crate::cruby::rb_cFloat), + (bits::Module, &raw const crate::cruby::rb_cModule), + (bits::NilClass, &raw const crate::cruby::rb_cNilClass), + (bits::Numeric, &raw const crate::cruby::rb_cNumeric), + (bits::Range, &raw const crate::cruby::rb_cRange), + (bits::Regexp, &raw const crate::cruby::rb_cRegexp), + (bits::Set, &raw const crate::cruby::rb_cSet), + (bits::Symbol, &raw const crate::cruby::rb_cSymbol), + (bits::String, &raw const crate::cruby::rb_cString), + (bits::TrueClass, &raw const crate::cruby::rb_cTrueClass), + (bits::Object, &raw const crate::cruby::rb_cObject), + (bits::BasicObject, &raw const crate::cruby::rb_cBasicObject), + ]; } diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs index c18b2735be..d7327975ce 100644 --- a/zjit/src/hir_type/mod.rs +++ b/zjit/src/hir_type/mod.rs @@ -1,12 +1,15 @@ +//! High-level intermediate representation types. + #![allow(non_upper_case_globals)] -use crate::cruby::{Qfalse, Qnil, Qtrue, VALUE, RUBY_T_ARRAY, RUBY_T_STRING, RUBY_T_HASH, RUBY_T_CLASS, RUBY_T_MODULE}; -use crate::cruby::{rb_cInteger, rb_cFloat, rb_cArray, rb_cHash, rb_cString, rb_cSymbol, rb_cObject, rb_cTrueClass, rb_cFalseClass, rb_cNilClass, rb_cRange, rb_cSet, rb_cRegexp, rb_cClass, rb_cModule, rb_zjit_singleton_class_p}; +use crate::cruby; +use crate::cruby::{rb_block_param_proxy, Qfalse, Qnil, Qtrue, RUBY_T_ARRAY, RUBY_T_HASH, RUBY_T_STRING, VALUE}; +use crate::cruby::{rb_cInteger, rb_cFloat, rb_cArray, rb_cHash, rb_cString, rb_cSymbol, rb_cRange, rb_zjit_singleton_class_p}; use crate::cruby::ClassRelationship; use crate::cruby::get_class_name; +use crate::cruby::get_module_name; use crate::cruby::ruby_sym_to_rust_string; use crate::cruby::rb_mRubyVMFrozenCore; -use crate::cruby::rb_obj_class; -use crate::hir::PtrPrintMap; +use crate::hir::{Const, PtrPrintMap}; use crate::profile::ProfiledType; #[derive(Copy, Clone, Debug, PartialEq)] @@ -73,7 +76,18 @@ fn write_spec(f: &mut std::fmt::Formatter, printer: &TypePrinter) -> std::fmt::R match ty.spec { Specialization::Any | Specialization::Empty => { Ok(()) }, Specialization::Object(val) if val == unsafe { rb_mRubyVMFrozenCore } => write!(f, "[VMFrozenCore]"), + Specialization::Object(val) if val == unsafe { rb_block_param_proxy } => write!(f, "[BlockParamProxy]"), Specialization::Object(val) if ty.is_subtype(types::Symbol) => write!(f, "[:{}]", ruby_sym_to_rust_string(val)), + Specialization::Object(val) if ty.is_subtype(types::Class) => + write!(f, "[{}@{:p}]", get_class_name(val), printer.ptr_map.map_ptr(val.0 as *const std::ffi::c_void)), + Specialization::Object(val) if ty.is_subtype(types::Module) => { + if let Some(name) = get_module_name(val) { + write!(f, "[{}@{:p}]", name, printer.ptr_map.map_ptr(val.0 as *const std::ffi::c_void)) + } else { + // Same as generic Specialization::Object + write!(f, "[{}]", val.print(printer.ptr_map)) + } + } Specialization::Object(val) => write!(f, "[{}]", val.print(printer.ptr_map)), // TODO(max): Ensure singleton classes never have Type specialization Specialization::Type(val) if unsafe { rb_zjit_singleton_class_p(val) } => @@ -84,14 +98,24 @@ fn write_spec(f: &mut std::fmt::Formatter, printer: &TypePrinter) -> std::fmt::R Specialization::TypeExact(val) => write!(f, "[class_exact:{}]", get_class_name(val)), Specialization::Int(val) if ty.is_subtype(types::CBool) => write!(f, "[{}]", val != 0), - Specialization::Int(val) if ty.is_subtype(types::CInt8) => write!(f, "[{}]", (val as i64) >> 56), - Specialization::Int(val) if ty.is_subtype(types::CInt16) => write!(f, "[{}]", (val as i64) >> 48), - Specialization::Int(val) if ty.is_subtype(types::CInt32) => write!(f, "[{}]", (val as i64) >> 32), + Specialization::Int(val) if ty.is_subtype(types::CInt8) => write!(f, "[{}]", (val & u8::MAX as u64) as i8), + Specialization::Int(val) if ty.is_subtype(types::CInt16) => write!(f, "[{}]", (val & u16::MAX as u64) as i16), + Specialization::Int(val) if ty.is_subtype(types::CInt32) => write!(f, "[{}]", (val & u32::MAX as u64) as i32), + Specialization::Int(val) if ty.is_subtype(types::CShape) => + write!(f, "[{:p}]", printer.ptr_map.map_shape(crate::cruby::ShapeId((val & u32::MAX as u64) as u32))), Specialization::Int(val) if ty.is_subtype(types::CInt64) => write!(f, "[{}]", val as i64), - Specialization::Int(val) if ty.is_subtype(types::CUInt8) => write!(f, "[{}]", val >> 56), - Specialization::Int(val) if ty.is_subtype(types::CUInt16) => write!(f, "[{}]", val >> 48), - Specialization::Int(val) if ty.is_subtype(types::CUInt32) => write!(f, "[{}]", val >> 32), - Specialization::Int(val) if ty.is_subtype(types::CUInt64) => write!(f, "[{}]", val), + Specialization::Int(val) if ty.is_subtype(types::CUInt8) => write!(f, "[{}]", val & u8::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt16) => write!(f, "[{}]", val & u16::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt32) => write!(f, "[{}]", val & u32::MAX as u64), + Specialization::Int(val) if ty.is_subtype(types::CUInt64) => { + // Print in hex if signed bit is set + if 0 != val & (1 << (u64::BITS - 1)) { + write!(f, "[0x{val:x}]") + } else { + write!(f, "[{val}]") + } + } + Specialization::Int(val) if ty.is_subtype(types::CPtr) => write!(f, "[{}]", Const::CPtr(val as *const u8).print(printer.ptr_map)), Specialization::Int(val) => write!(f, "[{val}]"), Specialization::Double(val) => write!(f, "[{val}]"), } @@ -153,18 +177,6 @@ fn is_range_exact(val: VALUE) -> bool { val.class_of() == unsafe { rb_cRange } } -fn is_module_exact(val: VALUE) -> bool { - if val.builtin_type() != RUBY_T_MODULE { - return false; - } - - // For Class and Module instances, `class_of` will return the singleton class of the object. - // Using `rb_obj_class` will give us the actual class of the module so we can check if the - // object is an instance of Module, or an instance of Module subclass. - let klass = unsafe { rb_obj_class(val) }; - klass == unsafe { rb_cModule } -} - impl Type { /// Create a `Type` from the given integer. pub const fn fixnum(val: i64) -> Type { @@ -174,10 +186,48 @@ impl Type { } } + fn bits_from_exact_class(class: VALUE) -> Option<u64> { + types::ExactBitsAndClass + .iter() + .find(|&(_, class_object)| unsafe { **class_object } == class) + .map(|&(bits, _)| bits) + } + + fn bits_from_subclass(class: VALUE) -> Option<u64> { + types::SubclassBitsAndClass + .iter() + .find(|&(_, class_object)| class.is_subclass_of(unsafe { **class_object }) == ClassRelationship::Subclass) + // Can't be an immediate if it's a subclass. + .map(|&(bits, _)| bits & !bits::Immediate) + } + + fn from_heap_object(val: VALUE) -> Type { + assert!(!val.special_const_p(), "val should be a heap object"); + let bits = + // GC-hidden types + if is_array_exact(val) { bits::ArrayExact } + else if is_hash_exact(val) { bits::HashExact } + else if is_string_exact(val) { bits::StringExact } + // Classes that have an immediate/heap split + else if val.class_of() == unsafe { rb_cInteger } { bits::Bignum } + else if val.class_of() == unsafe { rb_cFloat } { bits::HeapFloat } + else if val.class_of() == unsafe { rb_cSymbol } { bits::DynamicSymbol } + else if let Some(bits) = Self::bits_from_exact_class(val.class_of()) { bits } + else if let Some(bits) = Self::bits_from_subclass(val.class_of()) { bits } + else if val.data_p() { bits::TData } + else { + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(val.class_of())) + }; + let spec = Specialization::Object(val); + Type { bits, spec } + } + /// Create a `Type` from a Ruby `VALUE`. The type is not guaranteed to have object /// specialization in its `specialization` field (for example, `Qnil` will just be /// `types::NilClass`), but will be available via `ruby_object()`. pub fn from_value(val: VALUE) -> Type { + // Immediates if val.fixnum_p() { Type { bits: bits::Fixnum, spec: Specialization::Object(val) } } @@ -196,45 +246,27 @@ impl Type { // valid on imemo. Type { bits: bits::CallableMethodEntry, spec: Specialization::Object(val) } } - else if val.class_of() == unsafe { rb_cInteger } { - Type { bits: bits::Bignum, spec: Specialization::Object(val) } - } - else if val.class_of() == unsafe { rb_cFloat } { - Type { bits: bits::HeapFloat, spec: Specialization::Object(val) } - } - else if val.class_of() == unsafe { rb_cSymbol } { - Type { bits: bits::DynamicSymbol, spec: Specialization::Object(val) } - } - else if is_array_exact(val) { - Type { bits: bits::ArrayExact, spec: Specialization::Object(val) } - } - else if is_hash_exact(val) { - Type { bits: bits::HashExact, spec: Specialization::Object(val) } - } - else if is_range_exact(val) { - Type { bits: bits::RangeExact, spec: Specialization::Object(val) } - } - else if is_string_exact(val) { - Type { bits: bits::StringExact, spec: Specialization::Object(val) } - } - else if is_module_exact(val) { - Type { bits: bits::ModuleExact, spec: Specialization::Object(val) } - } - else if val.builtin_type() == RUBY_T_CLASS { - Type { bits: bits::Class, spec: Specialization::Object(val) } - } - else if val.class_of() == unsafe { rb_cRegexp } { - Type { bits: bits::RegexpExact, spec: Specialization::Object(val) } - } - else if val.class_of() == unsafe { rb_cSet } { - Type { bits: bits::SetExact, spec: Specialization::Object(val) } - } - else if val.class_of() == unsafe { rb_cObject } { - Type { bits: bits::ObjectExact, spec: Specialization::Object(val) } - } else { - // TODO(max): Add more cases for inferring type bits from built-in types - Type { bits: bits::BasicObject, spec: Specialization::Object(val) } + Self::from_heap_object(val) + } + } + + pub fn from_const(val: Const) -> Type { + match val { + Const::Value(v) => Self::from_value(v), + Const::CBool(v) => Self::from_cbool(v), + Const::CInt8(v) => Self::from_cint(types::CInt8, v as i64), + Const::CInt16(v) => Self::from_cint(types::CInt16, v as i64), + Const::CInt32(v) => Self::from_cint(types::CInt32, v as i64), + Const::CInt64(v) => Self::from_cint(types::CInt64, v), + Const::CUInt8(v) => Self::from_cint(types::CUInt8, v as i64), + Const::CUInt16(v) => Self::from_cint(types::CUInt16, v as i64), + Const::CUInt32(v) => Self::from_cint(types::CUInt32, v as i64), + Const::CAttrIndex(v) => Self::from_cint(types::CAttrIndex, v as i64), + Const::CShape(v) => Self::from_cint(types::CShape, v.0 as i64), + Const::CUInt64(v) => Self::from_cint(types::CUInt64, v as i64), + Const::CPtr(v) => Self::from_cptr(v), + Const::CDouble(v) => Self::from_double(v), } } @@ -245,11 +277,27 @@ impl Type { else if val.is_nil() { types::NilClass } else if val.is_true() { types::TrueClass } else if val.is_false() { types::FalseClass } - else if val.class() == unsafe { rb_cString } { types::StringExact } - else { - // TODO(max): Add more cases for inferring type bits from built-in types - Type { bits: bits::HeapObject, spec: Specialization::TypeExact(val.class()) } + // TODO(max): Revisit and maybe specialize to *not* an immediate + else { Self::from_class(val.class()).intersection(types::HeapBasicObject) } + } + + pub fn from_class(class: VALUE) -> Type { + if let Some(bits) = Self::bits_from_exact_class(class) { + return Type::from_bits(bits); + } + if let Some(bits) = Self::bits_from_subclass(class) { + return Type { bits, spec: Specialization::TypeExact(class) } } + unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", + get_class_name(class)) + } + + pub fn from_class_inexact(class: VALUE) -> Type { + let bits = types::InexactBitsAndClass + .iter() + .find(|&(_, class_object)| class.is_subclass_of(unsafe { **class_object }) == ClassRelationship::Subclass) + .unwrap_or_else(|| panic!("Class {} is not a subclass of BasicObject! Don't know what to do.", get_class_name(class))).0; + Type { bits, spec: Specialization::Type(class) } } /// Private. Only for creating type globals. @@ -275,6 +323,10 @@ impl Type { Type { bits: ty.bits, spec: Specialization::Int(val as u64) } } + pub fn from_cptr(val: *const u8) -> Type { + Type { bits: bits::CPtr, spec: Specialization::Int(val as u64) } + } + /// Create a `Type` (a `CDouble` with double specialization) from a f64. pub fn from_double(val: f64) -> Type { Type { bits: bits::CDouble, spec: Specialization::Double(val) } @@ -295,6 +347,25 @@ impl Type { self.is_subtype(types::NilClass) || self.is_subtype(types::FalseClass) } + pub fn has_value(&self, val: Const) -> bool { + match (self.spec, val) { + (Specialization::Object(v1), Const::Value(v2)) => v1 == v2, + (Specialization::Int(v1), Const::CBool(v2)) if self.is_subtype(types::CBool) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt8(v2)) if self.is_subtype(types::CInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt16(v2)) if self.is_subtype(types::CInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt32(v2)) if self.is_subtype(types::CInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CInt64(v2)) if self.is_subtype(types::CInt64) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt8(v2)) if self.is_subtype(types::CUInt8) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt16(v2)) if self.is_subtype(types::CUInt16) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CUInt32(v2)) if self.is_subtype(types::CUInt32) => v1 == (v2 as u64), + (Specialization::Int(v1), Const::CShape(v2)) if self.is_subtype(types::CShape) => v1 == (v2.0 as u64), + (Specialization::Int(v1), Const::CUInt64(v2)) if self.is_subtype(types::CUInt64) => v1 == v2, + (Specialization::Int(v1), Const::CPtr(v2)) if self.is_subtype(types::CPtr) => v1 == (v2 as u64), + (Specialization::Double(v1), Const::CDouble(v2)) => v1.to_bits() == v2.to_bits(), + _ => false, + } + } + /// Return the object specialization, if any. pub fn ruby_object(&self) -> Option<VALUE> { match self.spec { @@ -336,27 +407,54 @@ impl Type { } } + pub fn cint64_value(&self) -> Option<i64> { + match (self.is_subtype(types::CInt64), &self.spec) { + (true, Specialization::Int(val)) => Some(*val as i64), + _ => None, + } + } + + fn int_spec_signed(&self) -> Option<i64> { + assert!(self.is_subtype(types::CSigned), "int_spec_signed() only makes sense for signed integer types"); + match self.spec { + Specialization::Int(val) => Some(val as i64), + _ => None, + } + } + + pub fn known_nonnegative(&self) -> bool { + assert!(self.is_subtype(types::CSigned), "nonnegative() only makes sense for signed integer types"); + self.int_spec_signed().map_or(false, |val| val >= 0) + } + /// Return true if the Type has object specialization and false otherwise. pub fn ruby_object_known(&self) -> bool { matches!(self.spec, Specialization::Object(_)) } + /// Find a `T_*` type that is exactly as wide as `self`. + pub fn builtin_type_equivalent(&self) -> Option<cruby::ruby_value_type> { + if self.bit_equal(types::Array) { + Some(cruby::RUBY_T_ARRAY) + } else if self.bit_equal(types::Class) { + Some(cruby::RUBY_T_CLASS) + } else if self.bit_equal(types::Module) { + Some(cruby::RUBY_T_MODULE) + } else if self.bit_equal(types::String) { + Some(cruby::RUBY_T_STRING) + } else if self.bit_equal(types::Hash) { + Some(cruby::RUBY_T_HASH) + } else if self.bit_equal(types::TData) { + Some(cruby::RUBY_T_DATA) + } else { + None + } + } + fn is_builtin(class: VALUE) -> bool { - if class == unsafe { rb_cArray } { return true; } - if class == unsafe { rb_cClass } { return true; } - if class == unsafe { rb_cFalseClass } { return true; } - if class == unsafe { rb_cFloat } { return true; } - if class == unsafe { rb_cHash } { return true; } - if class == unsafe { rb_cInteger } { return true; } - if class == unsafe { rb_cModule } { return true; } - if class == unsafe { rb_cNilClass } { return true; } - if class == unsafe { rb_cObject } { return true; } - if class == unsafe { rb_cRange } { return true; } - if class == unsafe { rb_cRegexp } { return true; } - if class == unsafe { rb_cString } { return true; } - if class == unsafe { rb_cSymbol } { return true; } - if class == unsafe { rb_cTrueClass } { return true; } - false + types::ExactBitsAndClass + .iter() + .any(|&(_, class_object)| unsafe { *class_object } == class) } /// Union both types together, preserving specialization if possible. @@ -452,22 +550,10 @@ impl Type { if let Some(val) = self.exact_ruby_class() { return Some(val); } - if self.is_subtype(types::ArrayExact) { return Some(unsafe { rb_cArray }); } - if self.is_subtype(types::Class) { return Some(unsafe { rb_cClass }); } - if self.is_subtype(types::FalseClass) { return Some(unsafe { rb_cFalseClass }); } - if self.is_subtype(types::Float) { return Some(unsafe { rb_cFloat }); } - if self.is_subtype(types::HashExact) { return Some(unsafe { rb_cHash }); } - if self.is_subtype(types::Integer) { return Some(unsafe { rb_cInteger }); } - if self.is_subtype(types::ModuleExact) { return Some(unsafe { rb_cModule }); } - if self.is_subtype(types::NilClass) { return Some(unsafe { rb_cNilClass }); } - if self.is_subtype(types::ObjectExact) { return Some(unsafe { rb_cObject }); } - if self.is_subtype(types::RangeExact) { return Some(unsafe { rb_cRange }); } - if self.is_subtype(types::RegexpExact) { return Some(unsafe { rb_cRegexp }); } - if self.is_subtype(types::SetExact) { return Some(unsafe { rb_cSet }); } - if self.is_subtype(types::StringExact) { return Some(unsafe { rb_cString }); } - if self.is_subtype(types::Symbol) { return Some(unsafe { rb_cSymbol }); } - if self.is_subtype(types::TrueClass) { return Some(unsafe { rb_cTrueClass }); } - None + types::ExactBitsAndClass + .iter() + .find(|&(bits, _)| self.is_subtype(Type::from_bits(*bits))) + .map(|&(_, class_object)| unsafe { *class_object }) } /// Check bit equality of two `Type`s. Do not use! You are probably looking for [`Type::is_subtype`]. @@ -504,6 +590,22 @@ impl Type { pub fn print(self, ptr_map: &PtrPrintMap) -> TypePrinter<'_> { TypePrinter { inner: self, ptr_map } } + + pub fn num_bits(&self) -> u8 { + self.num_bytes() * crate::cruby::BITS_PER_BYTE as u8 + } + + pub fn num_bytes(&self) -> u8 { + if self.is_subtype(types::CUInt8) || self.is_subtype(types::CInt8) { return 1; } + if self.is_subtype(types::CUInt16) || self.is_subtype(types::CInt16) { return 2; } + if self.is_subtype(types::CUInt32) || self.is_subtype(types::CInt32) { return 4; } + if self.is_subtype(types::CShape) { + use crate::cruby::{SHAPE_ID_NUM_BITS, BITS_PER_BYTE}; + return (SHAPE_ID_NUM_BITS as usize / BITS_PER_BYTE).try_into().unwrap(); + } + // CUInt64, CInt64, CPtr, CNull, CDouble, or anything else defaults to 8 bytes + crate::cruby::SIZEOF_VALUE as u8 + } } #[cfg(test)] @@ -515,6 +617,11 @@ mod tests { use crate::cruby::rb_hash_new; use crate::cruby::rb_float_new; use crate::cruby::define_class; + use crate::cruby::rb_cObject; + use crate::cruby::rb_cSet; + use crate::cruby::rb_cTrueClass; + use crate::cruby::rb_cFalseClass; + use crate::cruby::rb_cNilClass; #[track_caller] fn assert_bit_equal(left: Type, right: Type) { @@ -557,6 +664,33 @@ mod tests { } #[test] + fn from_const() { + let cint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int(12)); + assert_eq!(format!("{}", cint32), "CInt32[12]"); + + let cint32 = Type::from_const(Const::CInt32(-12)); + assert_subtype(cint32, types::CInt32); + assert_eq!(cint32.spec, Specialization::Int((-12i64) as u64)); + assert_eq!(format!("{}", cint32), "CInt32[-12]"); + + let cuint32 = Type::from_const(Const::CInt32(12)); + assert_subtype(cuint32, types::CInt32); + assert_eq!(cuint32.spec, Specialization::Int(12)); + + let cuint32 = Type::from_const(Const::CUInt32(0xffffffff)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xffffffff)); + assert_eq!(format!("{}", cuint32), "CUInt32[4294967295]"); + + let cuint32 = Type::from_const(Const::CUInt32(0xc00087)); + assert_subtype(cuint32, types::CUInt32); + assert_eq!(cuint32.spec, Specialization::Int(0xc00087)); + assert_eq!(format!("{}", cuint32), "CUInt32[12583047]"); + } + + #[test] fn integer() { assert_subtype(Type::fixnum(123), types::Fixnum); assert_subtype(Type::fixnum(123), Type::fixnum(123)); @@ -573,6 +707,17 @@ mod tests { } #[test] + fn numeric() { + assert_subtype(types::Integer, types::Numeric); + assert_subtype(types::Float, types::Numeric); + assert_subtype(types::Float.union(types::Integer), types::Numeric); + assert_bit_equal(types::Float + .union(types::Integer) + .union(types::NumericExact) + .union(types::NumericSubclass), types::Numeric); + } + + #[test] fn symbol() { assert_subtype(types::StaticSymbol, types::Symbol); assert_subtype(types::DynamicSymbol, types::Symbol); @@ -594,6 +739,32 @@ mod tests { } #[test] + fn heap_basic_object() { + assert_not_subtype(Type::fixnum(123), types::HeapBasicObject); + assert_not_subtype(types::Fixnum, types::HeapBasicObject); + assert_subtype(types::Bignum, types::HeapBasicObject); + assert_not_subtype(types::Integer, types::HeapBasicObject); + assert_not_subtype(types::NilClass, types::HeapBasicObject); + assert_not_subtype(types::TrueClass, types::HeapBasicObject); + assert_not_subtype(types::FalseClass, types::HeapBasicObject); + assert_not_subtype(types::StaticSymbol, types::HeapBasicObject); + assert_subtype(types::DynamicSymbol, types::HeapBasicObject); + assert_not_subtype(types::Flonum, types::HeapBasicObject); + assert_subtype(types::HeapFloat, types::HeapBasicObject); + assert_not_subtype(types::BasicObject, types::HeapBasicObject); + assert_not_subtype(types::Object, types::HeapBasicObject); + assert_not_subtype(types::Immediate, types::HeapBasicObject); + assert_not_subtype(types::HeapBasicObject, types::Immediate); + crate::cruby::with_rubyvm(|| { + let left = Type::from_value(rust_str_to_ruby("hello")); + let right = Type::from_value(rust_str_to_ruby("world")); + assert_subtype(left, types::HeapBasicObject); + assert_subtype(right, types::HeapBasicObject); + assert_subtype(left.union(right), types::HeapBasicObject); + }); + } + + #[test] fn heap_object() { assert_not_subtype(Type::fixnum(123), types::HeapObject); assert_not_subtype(types::Fixnum, types::HeapObject); @@ -638,7 +809,7 @@ mod tests { #[test] fn integer_has_exact_ruby_class() { - assert_eq!(Type::fixnum(3).exact_ruby_class(), Some(unsafe { rb_cInteger }.into())); + assert_eq!(Type::fixnum(3).exact_ruby_class(), Some(unsafe { rb_cInteger })); assert_eq!(types::Fixnum.exact_ruby_class(), None); assert_eq!(types::Integer.exact_ruby_class(), None); } @@ -664,10 +835,27 @@ mod tests { } #[test] + fn from_class() { + crate::cruby::with_rubyvm(|| { + assert_bit_equal(Type::from_class(unsafe { rb_cInteger }), types::Integer); + assert_bit_equal(Type::from_class(unsafe { rb_cString }), types::StringExact); + assert_bit_equal(Type::from_class(unsafe { rb_cArray }), types::ArrayExact); + assert_bit_equal(Type::from_class(unsafe { rb_cHash }), types::HashExact); + assert_bit_equal(Type::from_class(unsafe { rb_cNilClass }), types::NilClass); + assert_bit_equal(Type::from_class(unsafe { rb_cTrueClass }), types::TrueClass); + assert_bit_equal(Type::from_class(unsafe { rb_cFalseClass }), types::FalseClass); + let c_class = define_class("C", unsafe { rb_cObject }); + assert_bit_equal(Type::from_class(c_class), Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class) }); + }); + } + + #[test] fn integer_has_ruby_class() { - assert_eq!(Type::fixnum(3).inexact_ruby_class(), Some(unsafe { rb_cInteger }.into())); - assert_eq!(types::Fixnum.inexact_ruby_class(), None); - assert_eq!(types::Integer.inexact_ruby_class(), None); + crate::cruby::with_rubyvm(|| { + assert_eq!(Type::fixnum(3).inexact_ruby_class(), Some(unsafe { rb_cInteger })); + assert_eq!(types::Fixnum.inexact_ruby_class(), None); + assert_eq!(types::Integer.inexact_ruby_class(), None); + }); } #[test] @@ -695,7 +883,7 @@ mod tests { assert_eq!(format!("{}", Type::from_cint(types::CInt32, -1)), "CInt32[-1]"); assert_eq!(format!("{}", Type::from_cint(types::CUInt32, -1)), "CUInt32[4294967295]"); assert_eq!(format!("{}", Type::from_cint(types::CInt64, -1)), "CInt64[-1]"); - assert_eq!(format!("{}", Type::from_cint(types::CUInt64, -1)), "CUInt64[18446744073709551615]"); + assert_eq!(format!("{}", Type::from_cint(types::CUInt64, -1)), "CUInt64[0xffffffffffffffff]"); assert_eq!(format!("{}", Type::from_cbool(true)), "CBool[true]"); assert_eq!(format!("{}", Type::from_cbool(false)), "CBool[false]"); assert_eq!(format!("{}", types::Fixnum), "Fixnum"); @@ -809,6 +997,17 @@ mod tests { } #[test] + fn string_subclass_is_string_subtype() { + crate::cruby::with_rubyvm(|| { + assert_subtype(types::StringExact, types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::String); + assert_subtype(Type::from_class(unsafe { rb_cString }), types::StringExact); + let c_class = define_class("C", unsafe { rb_cString }); + assert_subtype(Type::from_class(c_class), types::String); + }); + } + + #[test] fn union_specialized_with_no_relation_returns_unspecialized() { crate::cruby::with_rubyvm(|| { let string = Type::from_value(rust_str_to_ruby("hello")); @@ -829,4 +1028,80 @@ mod tests { assert_bit_equal(d_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); }); } + + #[test] + fn has_value() { + // With known values + crate::cruby::with_rubyvm(|| { + let a = rust_str_to_sym("a"); + let b = rust_str_to_sym("b"); + let ty = Type::from_value(a); + assert!(ty.has_value(Const::Value(a))); + assert!(!ty.has_value(Const::Value(b))); + }); + + let true_ty = Type::from_cbool(true); + assert!(true_ty.has_value(Const::CBool(true))); + assert!(!true_ty.has_value(Const::CBool(false))); + + let int8_ty = Type::from_cint(types::CInt8, 42); + assert!(int8_ty.has_value(Const::CInt8(42))); + assert!(!int8_ty.has_value(Const::CInt8(-1))); + let neg_int8_ty = Type::from_cint(types::CInt8, -1); + assert!(neg_int8_ty.has_value(Const::CInt8(-1))); + + let int16_ty = Type::from_cint(types::CInt16, 1000); + assert!(int16_ty.has_value(Const::CInt16(1000))); + assert!(!int16_ty.has_value(Const::CInt16(2000))); + + let int32_ty = Type::from_cint(types::CInt32, 100000); + assert!(int32_ty.has_value(Const::CInt32(100000))); + assert!(!int32_ty.has_value(Const::CInt32(-100000))); + + let int64_ty = Type::from_cint(types::CInt64, i64::MAX); + assert!(int64_ty.has_value(Const::CInt64(i64::MAX))); + assert!(!int64_ty.has_value(Const::CInt64(0))); + + let uint8_ty = Type::from_cint(types::CUInt8, u8::MAX as i64); + assert!(uint8_ty.has_value(Const::CUInt8(u8::MAX))); + assert!(!uint8_ty.has_value(Const::CUInt8(0))); + + let uint16_ty = Type::from_cint(types::CUInt16, u16::MAX as i64); + assert!(uint16_ty.has_value(Const::CUInt16(u16::MAX))); + assert!(!uint16_ty.has_value(Const::CUInt16(1))); + + let uint32_ty = Type::from_cint(types::CUInt32, u32::MAX as i64); + assert!(uint32_ty.has_value(Const::CUInt32(u32::MAX))); + assert!(!uint32_ty.has_value(Const::CUInt32(42))); + + let uint64_ty = Type::from_cint(types::CUInt64, i64::MAX); + assert!(uint64_ty.has_value(Const::CUInt64(i64::MAX as u64))); + assert!(!uint64_ty.has_value(Const::CUInt64(123))); + + let shape_ty = Type::from_cint(types::CShape, 0x1234); + assert!(shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + assert!(!shape_ty.has_value(Const::CShape(crate::cruby::ShapeId(0x5678)))); + + let ptr = 0x1000 as *const u8; + let ptr_ty = Type::from_cptr(ptr); + assert!(ptr_ty.has_value(Const::CPtr(ptr))); + assert!(!ptr_ty.has_value(Const::CPtr(0x2000 as *const u8))); + + let double_ty = Type::from_double(std::f64::consts::PI); + assert!(double_ty.has_value(Const::CDouble(std::f64::consts::PI))); + assert!(!double_ty.has_value(Const::CDouble(3.123))); + + let nan_ty = Type::from_double(f64::NAN); + assert!(nan_ty.has_value(Const::CDouble(f64::NAN))); + + // Mismatched types + assert!(!int8_ty.has_value(Const::CInt16(42))); + assert!(!int16_ty.has_value(Const::CInt32(1000))); + assert!(!uint8_ty.has_value(Const::CInt8(-1i8))); + + // Wrong specialization (unknown value) + assert!(!types::CInt8.has_value(Const::CInt8(42))); + assert!(!types::CBool.has_value(Const::CBool(true))); + assert!(!types::CShape.has_value(Const::CShape(crate::cruby::ShapeId(0x1234)))); + } } diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index 14fea76d1b..0fa800755d 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -1,27 +1,49 @@ +//! Code invalidation and patching for speculative optimizations. + use std::{collections::{HashMap, HashSet}, mem}; -use crate::{backend::lir::{asm_comment, Assembler}, cruby::{rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock, IseqPtr, RedefinitionFlag, ID, VALUE}, gc::IseqPayload, hir::Invariant, options::debug, state::{zjit_enabled_p, ZJITState}, virtualmem::CodePtr}; +use crate::{backend::lir::{Assembler, asm_comment}, cruby::{ID, IseqPtr, RedefinitionFlag, VALUE, iseq_name, rb_callable_method_entry_t, rb_gc_location, ruby_basic_operators, src_loc, with_vm_lock}, hir::Invariant, options::debug, state::{ZJITState, zjit_enabled_p, trace_invalidation}, virtualmem::CodePtr}; +use crate::payload::{IseqVersionRef, get_or_create_iseq_payload}; +use crate::codegen::invalidate_iseq_version; +use crate::cruby::rb_iseq_reset_jit_func; use crate::stats::with_time_stat; use crate::stats::Counter::invalidation_time_ns; use crate::gc::remove_gc_offsets; macro_rules! compile_patch_points { - ($cb:expr, $patch_points:expr, $($comment_args:tt)*) => { - with_time_stat(invalidation_time_ns, || { + ($cb:expr, $patch_points:expr, $cause:ident, $($comment_args:tt)*) => { + trace_invalidation(&format!($($comment_args)*), || with_time_stat(invalidation_time_ns, || { for patch_point in $patch_points { let written_range = $cb.with_write_ptr(patch_point.patch_point_ptr, |cb| { let mut asm = Assembler::new(); + asm.new_block_without_id("invalidation"); asm_comment!(asm, $($comment_args)*); asm.jmp(patch_point.side_exit_ptr.into()); asm.compile(cb).expect("can write existing code"); }); // Stop marking GC offsets corrupted by the jump instruction - remove_gc_offsets(patch_point.payload_ptr, &written_range); + remove_gc_offsets(patch_point.version, &written_range); + + let mut version = patch_point.version; + let iseq = unsafe { version.as_ref() }.iseq; + if !iseq.is_null() { + invalidate_iseq_version($cb, iseq, &mut version); + // Remember NoSingletonClass busts on the payload + if is_no_singleton_class!($cause) { + let payload = get_or_create_iseq_payload(iseq); + payload.was_invalidated_for_singleton_class_creation = true; + } + } } - }); + })); }; } +macro_rules! is_no_singleton_class { + (NoSingletonClass) => { true }; + ($_:ident) => { false }; +} + /// When a PatchPoint is invalidated, it generates a jump instruction from `from` to `to`. #[derive(Debug, Eq, Hash, PartialEq)] struct PatchPoint { @@ -29,8 +51,19 @@ struct PatchPoint { patch_point_ptr: CodePtr, /// Code pointer to a side exit side_exit_ptr: CodePtr, - /// Raw pointer to the ISEQ payload - payload_ptr: *mut IseqPayload, + /// ISEQ version to be invalidated + version: IseqVersionRef, +} + +impl PatchPoint { + /// PatchPointer constructor + fn new(patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, version: IseqVersionRef) -> PatchPoint { + Self { + patch_point_ptr, + side_exit_ptr, + version, + } + } } /// Used to track all of the various block references that contain assumptions @@ -40,8 +73,8 @@ pub struct Invariants { /// Set of ISEQs that are known to escape EP ep_escape_iseqs: HashSet<IseqPtr>, - /// Set of ISEQs whose JIT code assumes that it doesn't escape EP - no_ep_escape_iseqs: HashSet<IseqPtr>, + /// Map from ISEQ that's assumed to not escape EP to a set of patch points + no_ep_escape_iseq_patch_points: HashMap<IseqPtr, HashSet<PatchPoint>>, /// Map from a class and its associated basic operator to a set of patch points bop_patch_points: HashMap<(RedefinitionFlag, ruby_basic_operators), HashSet<PatchPoint>>, @@ -52,32 +85,94 @@ pub struct Invariants { /// Map from constant ID to patch points that assume the constant hasn't been redefined constant_state_patch_points: HashMap<ID, HashSet<PatchPoint>>, + /// Set of patch points that assume that the TracePoint is not enabled + no_trace_point_patch_points: HashSet<PatchPoint>, + /// Set of patch points that assume that the interpreter is running with only one ractor single_ractor_patch_points: HashSet<PatchPoint>, + + /// Map from a class to a set of patch points that assume objects of the class + /// will have no singleton class. + no_singleton_class_patch_points: HashMap<VALUE, HashSet<PatchPoint>>, + + /// Set of patch points that assume only the root box is active + root_box_patch_points: HashSet<PatchPoint>, + + /// Whether a non-root box has ever been created + non_root_box_created: bool, } impl Invariants { /// Update object references in Invariants pub fn update_references(&mut self) { - Self::update_iseq_references(&mut self.ep_escape_iseqs); - Self::update_iseq_references(&mut self.no_ep_escape_iseqs); + self.update_ep_escape_iseqs(); + self.update_no_ep_escape_iseq_patch_points(); + self.update_cme_patch_points(); + self.update_no_singleton_class_patch_points(); } - /// Update ISEQ references in a given HashSet<IseqPtr> - fn update_iseq_references(iseqs: &mut HashSet<IseqPtr>) { - let mut moved: Vec<IseqPtr> = Vec::with_capacity(iseqs.len()); + /// Forget an ISEQ when freeing it. We need to because a) if the address is reused, we'd be + /// tracking the wrong object b) dead VALUEs in the table can means we risk passing invalid + /// VALUEs to `rb_gc_location()`. + pub fn forget_iseq(&mut self, iseq: IseqPtr) { + // Why not patch the patch points? If the ISEQ is dead then the GC also proved that all + // generated code referencing the ISEQ are unreachable. We mark the ISEQs baked into + // generated code. + self.ep_escape_iseqs.remove(&iseq); + self.no_ep_escape_iseq_patch_points.remove(&iseq); + } - iseqs.retain(|&old_iseq| { - let new_iseq = unsafe { rb_gc_location(VALUE(old_iseq as usize)) }.0 as IseqPtr; - if old_iseq != new_iseq { - moved.push(new_iseq); - } - old_iseq == new_iseq - }); + /// Forget a CME when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_cme(&mut self, cme: *const rb_callable_method_entry_t) { + self.cme_patch_points.remove(&cme); + } - for new_iseq in moved { - iseqs.insert(new_iseq); - } + /// Forget a class when freeing it. See [Self::forget_iseq] for reasoning. + pub fn forget_klass(&mut self, klass: VALUE) { + self.no_singleton_class_patch_points.remove(&klass); + } + + /// Update ISEQ references in Invariants::ep_escape_iseqs + fn update_ep_escape_iseqs(&mut self) { + let updated = std::mem::take(&mut self.ep_escape_iseqs) + .into_iter() + .map(|iseq| unsafe { rb_gc_location(iseq.into()) }.as_iseq()) + .collect(); + self.ep_escape_iseqs = updated; + } + + /// Update ISEQ references in Invariants::no_ep_escape_iseq_patch_points + fn update_no_ep_escape_iseq_patch_points(&mut self) { + let updated = std::mem::take(&mut self.no_ep_escape_iseq_patch_points) + .into_iter() + .map(|(iseq, patch_points)| { + let new_iseq = unsafe { rb_gc_location(iseq.into()) }; + (new_iseq.as_iseq(), patch_points) + }) + .collect(); + self.no_ep_escape_iseq_patch_points = updated; + } + + fn update_cme_patch_points(&mut self) { + let updated_cme_patch_points = std::mem::take(&mut self.cme_patch_points) + .into_iter() + .map(|(cme, patch_points)| { + let new_cme = unsafe { rb_gc_location(cme.into()) }; + (new_cme.as_cme(), patch_points) + }) + .collect(); + self.cme_patch_points = updated_cme_patch_points; + } + + fn update_no_singleton_class_patch_points(&mut self) { + let updated_no_singleton_class_patch_points = std::mem::take(&mut self.no_singleton_class_patch_points) + .into_iter() + .map(|(klass, patch_points)| { + let new_klass = unsafe { rb_gc_location(klass) }; + (new_klass, patch_points) + }) + .collect(); + self.no_singleton_class_patch_points = updated_no_singleton_class_patch_points; } } @@ -93,13 +188,13 @@ pub extern "C" fn rb_zjit_bop_redefined(klass: RedefinitionFlag, bop: ruby_basic with_vm_lock(src_loc!(), || { let invariants = ZJITState::get_invariants(); - if let Some(patch_points) = invariants.bop_patch_points.get(&(klass, bop)) { + if let Some(patch_points) = invariants.bop_patch_points.remove(&(klass, bop)) { let cb = ZJITState::get_code_block(); let bop = Invariant::BOPRedefined { klass, bop }; debug!("BOP is redefined: {}", bop); // Invalidate all patch points for this BOP - compile_patch_points!(cb, patch_points, "BOP is redefined: {}", bop); + compile_patch_points!(cb, patch_points, BOP, "BOP is redefined: {}", bop); cb.mark_all_executable(); } @@ -109,28 +204,62 @@ pub extern "C" fn rb_zjit_bop_redefined(klass: RedefinitionFlag, bop: ruby_basic /// Invalidate blocks for a given ISEQ that assumes environment pointer is /// equal to base pointer. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_invalidate_ep_is_bp(iseq: IseqPtr) { +pub extern "C" fn rb_zjit_invalidate_no_ep_escape(iseq: IseqPtr) { // Skip tracking EP escapes on boot. We don't need to invalidate anything during boot. if !ZJITState::has_instance() { return; } - // Remember that this ISEQ may escape EP - let invariants = ZJITState::get_invariants(); - invariants.ep_escape_iseqs.insert(iseq); + with_vm_lock(src_loc!(), || { + // Remember that this ISEQ may escape EP + let invariants = ZJITState::get_invariants(); + invariants.ep_escape_iseqs.insert(iseq); - // If the ISEQ has been compiled assuming it doesn't escape EP, invalidate the JIT code. - // Note: Nobody calls track_no_ep_escape_assumption() for now, so this is always false. - // TODO: Add a PatchPoint that assumes EP == BP in HIR and invalidate it here. - if invariants.no_ep_escape_iseqs.contains(&iseq) { - unimplemented!("Invalidation on EP escape is not implemented yet"); - } + // If the ISEQ has been compiled assuming it doesn't escape EP, invalidate the JIT code. + if let Some(patch_points) = invariants.no_ep_escape_iseq_patch_points.remove(&iseq) { + debug!("EP is escaped: {}", iseq_name(iseq)); + + // Invalidate the patch points for this ISEQ + let cb = ZJITState::get_code_block(); + compile_patch_points!(cb, patch_points, EP, "EP is escaped: {}", iseq_name(iseq)); + + // Also invalidate the ISEQ version so the method falls back to the + // interpreter on the next call. NoEPEscape PatchPoint side exits use + // without_locals() and don't save locals to the frame. If a PatchPoint + // fires on a later call (where EP hasn't escaped), the interpreter would + // read stale locals (e.g., nil instead of [] for keyword defaults). + // + // We can't use invalidate_iseq_version() here because it skips when + // at MAX_ISEQ_VERSIONS (to prevent unbounded recompilation). Instead, + // directly mark the version as invalidated and reset jit_func so the + // interpreter takes over permanently. + let payload = crate::payload::get_or_create_iseq_payload(iseq); + if let Some(version) = payload.versions.last_mut() { + use crate::payload::IseqStatus; + if unsafe { version.as_ref() }.status != IseqStatus::Invalidated { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + unsafe { rb_iseq_reset_jit_func(iseq) }; + } + } + + cb.mark_all_executable(); + } + }); } /// Track that JIT code for a ISEQ will assume that base pointer is equal to environment pointer. -pub fn track_no_ep_escape_assumption(iseq: IseqPtr) { +pub fn track_no_ep_escape_assumption( + iseq: IseqPtr, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { let invariants = ZJITState::get_invariants(); - invariants.no_ep_escape_iseqs.insert(iseq); + invariants.no_ep_escape_iseq_patch_points.entry(iseq).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); } /// Returns true if a given ISEQ has previously escaped environment pointer. @@ -144,14 +273,14 @@ pub fn track_bop_assumption( bop: ruby_basic_operators, patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, - payload_ptr: *mut IseqPayload, + version: IseqVersionRef, ) { let invariants = ZJITState::get_invariants(); - invariants.bop_patch_points.entry((klass, bop)).or_default().insert(PatchPoint { + invariants.bop_patch_points.entry((klass, bop)).or_default().insert(PatchPoint::new( patch_point_ptr, side_exit_ptr, - payload_ptr, - }); + version, + )); } /// Track a patch point for a callable method entry (CME). @@ -159,14 +288,14 @@ pub fn track_cme_assumption( cme: *const rb_callable_method_entry_t, patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, - payload_ptr: *mut IseqPayload, + version: IseqVersionRef, ) { let invariants = ZJITState::get_invariants(); - invariants.cme_patch_points.entry(cme).or_default().insert(PatchPoint { + invariants.cme_patch_points.entry(cme).or_default().insert(PatchPoint::new( patch_point_ptr, side_exit_ptr, - payload_ptr, - }); + version, + )); } /// Track a patch point for each constant name in a constant path assumption. @@ -174,7 +303,7 @@ pub fn track_stable_constant_names_assumption( idlist: *const ID, patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, - payload_ptr: *mut IseqPayload, + version: IseqVersionRef, ) { let invariants = ZJITState::get_invariants(); @@ -185,16 +314,31 @@ pub fn track_stable_constant_names_assumption( break; } - invariants.constant_state_patch_points.entry(id).or_default().insert(PatchPoint { + invariants.constant_state_patch_points.entry(id).or_default().insert(PatchPoint::new( patch_point_ptr, side_exit_ptr, - payload_ptr, - }); + version, + )); idx += 1; } } +/// Track a patch point for objects of a given class will have no singleton class. +pub fn track_no_singleton_class_assumption( + klass: VALUE, + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_singleton_class_patch_points.entry(klass).or_default().insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + /// Called when a method is redefined. Invalidates all JIT code that depends on the CME. #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_cme_invalidate(cme: *const rb_callable_method_entry_t) { @@ -211,7 +355,7 @@ pub extern "C" fn rb_zjit_cme_invalidate(cme: *const rb_callable_method_entry_t) debug!("CME is invalidated: {:?}", cme); // Invalidate all patch points for this CME - compile_patch_points!(cb, patch_points, "CME is invalidated: {:?}", cme); + compile_patch_points!(cb, patch_points, CME, "CME is invalidated: {:?}", cme); cb.mark_all_executable(); } @@ -228,12 +372,12 @@ pub extern "C" fn rb_zjit_constant_state_changed(id: ID) { with_vm_lock(src_loc!(), || { let invariants = ZJITState::get_invariants(); - if let Some(patch_points) = invariants.constant_state_patch_points.get(&id) { + if let Some(patch_points) = invariants.constant_state_patch_points.remove(&id) { let cb = ZJITState::get_code_block(); debug!("Constant state changed: {:?}", id); // Invalidate all patch points for this constant ID - compile_patch_points!(cb, patch_points, "Constant state changed: {:?}", id); + compile_patch_points!(cb, patch_points, Const, "Constant state changed: {:?}", id); cb.mark_all_executable(); } @@ -241,13 +385,17 @@ pub extern "C" fn rb_zjit_constant_state_changed(id: ID) { } /// Track the JIT code that assumes that the interpreter is running with only one ractor -pub fn track_single_ractor_assumption(patch_point_ptr: CodePtr, side_exit_ptr: CodePtr, payload_ptr: *mut IseqPayload) { +pub fn track_single_ractor_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { let invariants = ZJITState::get_invariants(); - invariants.single_ractor_patch_points.insert(PatchPoint { + invariants.single_ractor_patch_points.insert(PatchPoint::new( patch_point_ptr, side_exit_ptr, - payload_ptr, - }); + version, + )); } /// Callback for when Ruby is about to spawn a ractor. In that case we need to @@ -264,8 +412,132 @@ pub extern "C" fn rb_zjit_before_ractor_spawn() { let patch_points = mem::take(&mut ZJITState::get_invariants().single_ractor_patch_points); // Invalidate all patch points for single ractor mode - compile_patch_points!(cb, patch_points, "Another ractor spawned, invalidating single ractor mode assumption"); + compile_patch_points!(cb, patch_points, Ractor, "Another ractor spawned, invalidating single ractor mode assumption"); + + cb.mark_all_executable(); + }); +} + +pub fn track_no_trace_point_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.no_trace_point_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_tracing_invalidate_all() { + use crate::payload::{get_or_create_iseq_payload, IseqStatus}; + use crate::cruby::for_each_iseq; + + if !zjit_enabled_p() { + return; + } + + // Stop other ractors since we are going to patch machine code. + with_vm_lock(src_loc!(), || { + debug!("Invalidating all ZJIT compiled code due to TracePoint"); + + for_each_iseq(|iseq| { + let payload = get_or_create_iseq_payload(iseq); + + if let Some(version) = payload.versions.last_mut() { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + } + unsafe { rb_iseq_reset_jit_func(iseq) }; + }); + + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut ZJITState::get_invariants().no_trace_point_patch_points); + + compile_patch_points!(cb, patch_points, TracePoint, "TracePoint is enabled, invalidating no TracePoint assumption"); cb.mark_all_executable(); }); } + +/// Track the JIT code that assumes only the root box is active +pub fn track_root_box_assumption( + patch_point_ptr: CodePtr, + side_exit_ptr: CodePtr, + version: IseqVersionRef, +) { + let invariants = ZJITState::get_invariants(); + invariants.root_box_patch_points.insert(PatchPoint::new( + patch_point_ptr, + side_exit_ptr, + version, + )); +} + +/// Returns true if a non-root box has ever been created. +pub fn non_root_box_created() -> bool { + ZJITState::get_invariants().non_root_box_created +} + +/// Callback for when a non-root box is created. In that case we need to +/// invalidate every block that assumes root-box-only mode. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_root_box() { + // If ZJIT isn't enabled, do nothing + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + invariants.non_root_box_created = true; + + let cb = ZJITState::get_code_block(); + let patch_points = mem::take(&mut invariants.root_box_patch_points); + + // Invalidate all patch points for root box mode + compile_patch_points!(cb, patch_points, Box, "Non-root box created, invalidating root box assumption"); + + cb.mark_all_executable(); + }); +} + +/// Returns true if we've seen a singleton class of a given class since boot. +/// This is used to avoid an invalidation loop where we repeatedly compile code +/// that assumes no singleton class, only to have it invalidated. +pub fn has_singleton_class_of(klass: VALUE) -> bool { + ZJITState::get_invariants() + .no_singleton_class_patch_points + .get(&klass) + .map_or(false, |patch_points| patch_points.is_empty()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_invalidate_no_singleton_class(klass: VALUE) { + if !zjit_enabled_p() { + return; + } + + with_vm_lock(src_loc!(), || { + let invariants = ZJITState::get_invariants(); + match invariants.no_singleton_class_patch_points.get_mut(&klass) { + Some(patch_points) => { + // Invalidate existing patch points and let has_singleton_class_of() + // return true when they are compiled again + let patch_points = mem::take(patch_points); + if !patch_points.is_empty() { + let cb = ZJITState::get_code_block(); + debug!("Singleton class created for {:?}", klass); + compile_patch_points!(cb, patch_points, NoSingletonClass, "Singleton class created for {:?}", klass); + cb.mark_all_executable(); + } + } + None => { + // Let has_singleton_class_of() return true for this class + invariants.no_singleton_class_patch_points.insert(klass, HashSet::new()); + } + } + }); +} diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs new file mode 100644 index 0000000000..8691833db0 --- /dev/null +++ b/zjit/src/jit_frame.rs @@ -0,0 +1,314 @@ +use crate::cruby::{IseqPtr, VALUE, rb_gc_mark_movable, rb_gc_location}; +use crate::cruby::zjit_jit_frame; +use crate::codegen::iseq_may_write_block_code; +use crate::state::ZJITState; + +/// JITFrame struct is defined in zjit.h (the single source of truth) and +/// imported into Rust via bindgen. See zjit.h for field documentation. +pub type JITFrame = zjit_jit_frame; + +impl JITFrame { + /// Allocate a JITFrame on the heap, register it with ZJITState, and return + /// a raw pointer that remains valid for the lifetime of the process. + fn alloc(jit_frame: JITFrame) -> *const Self { + let raw_ptr = Box::into_raw(Box::new(jit_frame)); + ZJITState::get_jit_frames().push(raw_ptr); + raw_ptr as *const _ + } + + /// Create a JITFrame for an ISEQ frame. + pub fn new_iseq(pc: *const VALUE, iseq: IseqPtr) -> *const Self { + let materialize_block_code = !iseq_may_write_block_code(iseq); + Self::alloc(JITFrame { pc, iseq, materialize_block_code }) + } + + /// Mark the iseq pointer for GC. Called from rb_zjit_root_mark. + pub fn mark(&self) { + if !self.iseq.is_null() { + unsafe { rb_gc_mark_movable(VALUE::from(self.iseq)); } + } + } + + /// Update the iseq pointer after GC compaction. + pub fn update_references(&mut self) { + if !self.iseq.is_null() { + let new_iseq = unsafe { rb_gc_location(VALUE::from(self.iseq)) }.as_iseq(); + if self.iseq != new_iseq { + self.iseq = new_iseq; + } + } + } +} + +/// Update the iseq pointer in an on-stack JITFrame during GC compaction. +/// Called from rb_execution_context_update in vm.c. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_frame_update_references(jit_frame: *mut JITFrame) { + unsafe { &mut *jit_frame }.update_references(); +} + +#[cfg(test)] +mod tests { + use crate::cruby::{eval, inspect}; + use insta::assert_snapshot; + + #[test] + fn test_jit_frame_entry_first() { + eval(r#" + def test + itself + callee + end + + def callee + caller + end + + test + "#); + assert_snapshot!(inspect("test.first"), @r#""<compiled>:4:in 'Object#test'""#); + } + + #[test] + fn test_materialize_one_frame() { + assert_snapshot!(inspect(" + def jit_entry + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + #[test] + fn test_materialize_two_frames() { // materialize caller frames on raise + // At the point of `resuce`, there are two lightweight frames on stack and both need to be + // materialized before passing control to interpreter. + assert_snapshot!(inspect(" + def jit_entry = raise_and_rescue + def raise_and_rescue + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + // Materialize frames on side exit: a type guard triggers a side exit with + // multiple JIT frames on the stack. All frames must be materialized before + // the interpreter resumes. + #[test] + fn test_side_exit_materialize_frames() { + assert_snapshot!(inspect(" + def side_exit(n) = 1 + n + def jit_frame(n) = 1 + side_exit(n) + def entry(n) = jit_frame(n) + entry(2) + [entry(2), entry(2.0)] + "), @"[4, 4.0]"); + } + + // BOP invalidation must not overwrite the top-most frame's PC with + // jit_frame's PC. After invalidation the interpreter resumes at a new + // PC, so a stale jit_frame PC would cause wrong execution. + #[test] + fn test_bop_invalidation() { + assert_snapshot!(inspect(r#" + def test + eval("class Integer; def +(_) = 100; end") + 1 + 2 + end + test + test + "#), @"100"); + } + + // Side exit at the very start of a method, before gen_save_pc_for_gc has + // updated the entry JITFrame. + #[test] + fn test_side_exit_before_jit_frame_update() { + assert_snapshot!(inspect(" + def entry(n) = n + 1 + entry(1) + [entry(1), entry(1.0)] + "), @"[2, 2.0]"); + } + + #[test] + fn test_caller_iseq() { + assert_snapshot!(inspect(r#" + def callee = call_caller + def test = callee + + def callee2 = call_caller + def test2 = callee2 + + def call_caller = caller + + test + test2 + test.first + "#), @r#""<compiled>:2:in 'Object#callee'""#); + } + + // ISEQ must be readable during exception handling so the interpreter + // can look up rescue/ensure tables. + #[test] + fn test_iseq_on_raise() { + assert_snapshot!(inspect(r#" + def jit_entry(v) = make_range_then_exit(v) + def make_range_then_exit(v) + range = (v..1) + super rescue range + end + jit_entry(0) + jit_entry(0) + jit_entry(0/1r) + "#), @"(0/1)..1"); + } + + // Multiple exception raises during keyword argument evaluation: each + // raise needs correct ISEQ for catch table lookup. + #[test] + fn test_iseq_on_raise_on_ensure() { + assert_snapshot!(inspect(r#" + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + "#), @r#"["a", "b", "c"]"#); + } + + // Send fallback (e.g. method_missing) calls into the interpreter, which + // reads cfp->iseq via GET_ISEQ(). gen_prepare_non_leaf_call writes the + // iseq to JITFrame, but GET_ISEQ reads cfp->iseq directly. This test + // ensures the interpreter can resolve the caller iseq for backtraces. + #[test] + fn test_send_fallback_caller_location() { + assert_snapshot!(inspect(r#" + def callee = caller_locations(1, 1)[0].label + def test = callee + test + test + "#), @r#""Object#test""#); + } + + // A send fallback may throw (e.g. via method_missing raising). The + // interpreter must be able to find the correct rescue handler in the + // caller's ISEQ catch table. This exercises throw through send fallback. + #[test] + fn test_send_fallback_throw() { + assert_snapshot!(inspect(r#" + class Foo + def method_missing(name, *) = raise("no #{name}") + end + def test + Foo.new.bar + rescue RuntimeError => e + e.message + end + test + test + "#), @r#""no bar""#); + } + + // Proc.new inside a block passed via invokeblock captures the caller's + // block_code. When the JIT compiles the caller, block_code must be + // correctly available for the proc to work. + #[test] + fn test_proc_from_invokeblock() { + assert_snapshot!(inspect(" + def capture_block(&blk) = blk + def test = capture_block { 42 } + test + test.call + "), @"42"); + } + + // binding() called from a JIT-compiled callee must see the correct + // source location (iseq + pc) of the caller frame. + #[test] + fn test_binding_source_location() { + assert_snapshot!(inspect(r#" + def callee = binding + def test = callee + test + b = test + b.source_location[1] > 0 + "#), @"true"); + } + + // $~ (Regexp special variable) is stored via svar which walks the EP + // chain to find the LEP. rb_vm_svar_lep uses rb_zjit_cfp_has_iseq to + // skip C frames, so it must work correctly with JITFrame. + #[test] + fn test_svar_regexp_match() { + assert_snapshot!(inspect(r#" + def test(s) + s =~ /hello/ + $~ + end + test("hello world") + test("hello world").to_s + "#), @r#""hello""#); + } + + // C function calls with rb_block_call (like Array#each, Enumerable#map) + // write an ifunc to cfp->block_code after the JIT pushes the C frame. + // GC must mark and relocate this ifunc. This test exercises the code + // path fixed by "Fix ZJIT segfault: write block_code for C frames and + // fix GC marking". + #[test] + fn test_cfunc_block_code_gc() { + assert_snapshot!(inspect(" + def test + # Use a cfunc that calls back into Ruby with a block (rb_block_call) + [1, 2, 3].map { |x| x.to_s } + end + test + test + "), @r#"["1", "2", "3"]"#); + } + + // Multiple levels of cfunc-with-block: a JIT-compiled method calls a + // cfunc that yields, and the block itself calls another cfunc that + // yields. Each C frame's block_code must be properly initialized. + #[test] + fn test_nested_cfunc_with_block() { + assert_snapshot!(inspect(" + def test + [1, 2].flat_map { |x| [x, x + 10].map { |y| y * 2 } } + end + test + test + "), @"[2, 22, 4, 24]"); + } +} diff --git a/zjit/src/json.rs b/zjit/src/json.rs new file mode 100644 index 0000000000..fa4b216821 --- /dev/null +++ b/zjit/src/json.rs @@ -0,0 +1,700 @@ +//! Single file JSON serializer for iongraph output of ZJIT HIR. + +use std::{ + fmt, + io::{self, Write}, +}; + +pub trait Jsonable { + fn to_json(&self) -> Json; +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Json { + Null, + Bool(bool), + Integer(isize), + UnsignedInteger(usize), + Floating(f64), + String(String), + Array(Vec<Json>), + Object(Vec<(String, Json)>), +} + +impl Json { + /// Convenience method for constructing a JSON array. + pub fn array<I, T>(iter: I) -> Self + where + I: IntoIterator<Item = T>, + T: Into<Json>, + { + Json::Array(iter.into_iter().map(Into::into).collect()) + } + + pub fn empty_array() -> Self { + Json::Array(Vec::new()) + } + + pub fn object() -> JsonObjectBuilder { + JsonObjectBuilder::new() + } + + pub fn marshal<W: Write>(&self, writer: &mut W) -> JsonResult<()> { + match self { + Json::Null => writer.write_all(b"null"), + Json::Bool(b) => writer.write_all(if *b { b"true" } else { b"false" }), + Json::Integer(i) => write!(writer, "{i}"), + Json::UnsignedInteger(u) => write!(writer, "{u}"), + Json::Floating(f) => write!(writer, "{f}"), + Json::String(s) => return Self::write_str(writer, s), + Json::Array(jsons) => return Self::write_array(writer, jsons), + Json::Object(map) => return Self::write_object(writer, map), + }?; + Ok(()) + } + + pub fn write_str<W: Write>(writer: &mut W, s: &str) -> JsonResult<()> { + writer.write_all(b"\"")?; + + for ch in s.chars() { + match ch { + '"' => write!(writer, "\\\"")?, + '\\' => write!(writer, "\\\\")?, + // The following characters are control, but have a canonical representation. + // https://datatracker.ietf.org/doc/html/rfc8259#section-7 + '\n' => write!(writer, "\\n")?, + '\r' => write!(writer, "\\r")?, + '\t' => write!(writer, "\\t")?, + '\x08' => write!(writer, "\\b")?, + '\x0C' => write!(writer, "\\f")?, + ch if ch.is_control() => { + let code_point = ch as u32; + write!(writer, "\\u{code_point:04X}")? + } + _ => write!(writer, "{ch}")?, + }; + } + + writer.write_all(b"\"")?; + Ok(()) + } + + pub fn write_array<W: Write>(writer: &mut W, jsons: &[Json]) -> JsonResult<()> { + writer.write_all(b"[")?; + let mut prefix = ""; + for item in jsons { + write!(writer, "{prefix}")?; + item.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"]")?; + Ok(()) + } + + pub fn write_object<W: Write>(writer: &mut W, pairs: &[(String, Json)]) -> JsonResult<()> { + writer.write_all(b"{")?; + let mut prefix = ""; + for (k, v) in pairs { + // Escape the keys, despite not being `Json::String` objects. + write!(writer, "{prefix}")?; + Self::write_str(writer, k)?; + writer.write_all(b":")?; + v.marshal(writer)?; + prefix = ", "; + } + writer.write_all(b"}")?; + Ok(()) + } +} + +impl std::fmt::Display for Json { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let mut buf = Vec::new(); + self.marshal(&mut buf).map_err(|_| std::fmt::Error)?; + let s = String::from_utf8(buf).map_err(|_| std::fmt::Error)?; + write!(f, "{s}") + } +} + +pub struct JsonObjectBuilder { + pairs: Vec<(String, Json)>, +} + +impl JsonObjectBuilder { + pub fn new() -> Self { + Self { pairs: Vec::new() } + } + + pub fn insert<K, V>(mut self, key: K, value: V) -> Self + where + K: Into<String>, + V: Into<Json>, + { + self.pairs.push((key.into(), value.into())); + self + } + + pub fn build(self) -> Json { + Json::Object(self.pairs) + } +} + +impl From<&str> for Json { + fn from(s: &str) -> Json { + Json::String(s.to_string()) + } +} + +impl From<String> for Json { + fn from(s: String) -> Json { + Json::String(s) + } +} + +impl From<i32> for Json { + fn from(i: i32) -> Json { + Json::Integer(i as isize) + } +} + +impl From<i64> for Json { + fn from(i: i64) -> Json { + Json::Integer(i as isize) + } +} + +impl From<u32> for Json { + fn from(u: u32) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<u64> for Json { + fn from(u: u64) -> Json { + Json::UnsignedInteger(u as usize) + } +} + +impl From<usize> for Json { + fn from(u: usize) -> Json { + Json::UnsignedInteger(u) + } +} + +impl From<bool> for Json { + fn from(b: bool) -> Json { + Json::Bool(b) + } +} + +impl TryFrom<f64> for Json { + type Error = JsonError; + fn try_from(f: f64) -> Result<Self, Self::Error> { + if f.is_finite() { + Ok(Json::Floating(f)) + } else { + Err(JsonError::FloatError(f)) + } + } +} + +impl<T: Into<Json>> From<Vec<T>> for Json { + fn from(v: Vec<T>) -> Json { + Json::Array(v.into_iter().map(|item| item.into()).collect()) + } +} + +/// Convenience type for a result in JSON serialization. +pub type JsonResult<W> = std::result::Result<W, JsonError>; + +#[derive(Debug)] +pub enum JsonError { + /// Wrapper for a standard `io::Error`. + IoError(io::Error), + /// On attempting to serialize an invalid `f32` or `f64`. + /// Stores invalid values as 64 bit float. + FloatError(f64), +} + +impl From<io::Error> for JsonError { + fn from(err: io::Error) -> Self { + JsonError::IoError(err) + } +} + +impl fmt::Display for JsonError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JsonError::FloatError(v) => write!(f, "Cannot serialize float {v}"), + JsonError::IoError(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for JsonError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + JsonError::IoError(e) => Some(e), + JsonError::FloatError(_) => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use insta::assert_snapshot; + + fn marshal_to_string(json: &Json) -> String { + let mut buf = Vec::new(); + json.marshal(&mut buf).unwrap(); + String::from_utf8(buf).unwrap() + } + + #[test] + fn test_null() { + let json = Json::Null; + assert_snapshot!(marshal_to_string(&json), @"null"); + } + + #[test] + fn test_bool() { + let json: Json = true.into(); + assert_snapshot!(marshal_to_string(&json), @"true"); + let json: Json = false.into(); + assert_snapshot!(marshal_to_string(&json), @"false"); + } + + #[test] + fn test_integer_positive() { + let json: Json = 42.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_integer_negative() { + let json: Json = (-123).into(); + assert_snapshot!(marshal_to_string(&json), @"-123"); + } + + #[test] + fn test_integer_zero() { + let json: Json = 0.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_floating() { + let json = 2.14159.try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"2.14159"); + } + + #[test] + fn test_floating_negative() { + let json = (-2.5).try_into(); + assert!(json.is_ok()); + let json = json.unwrap(); + assert_snapshot!(marshal_to_string(&json), @"-2.5"); + } + + #[test] + fn test_floating_error() { + let json: Result<Json, JsonError> = f64::NAN.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + + let json: Result<Json, JsonError> = f64::NEG_INFINITY.try_into(); + assert!(matches!(json, Err(JsonError::FloatError(_)))); + } + + #[test] + fn test_string_simple() { + let json: Json = "hello".into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello""#); + } + + #[test] + fn test_string_empty() { + let json: Json = "".into(); + assert_snapshot!(marshal_to_string(&json), @r#""""#); + } + + #[test] + fn test_string_with_quotes() { + let json: Json = r#"hello "world""#.into(); + assert_snapshot!(marshal_to_string(&json), @r#""hello \"world\"""#); + } + + #[test] + fn test_string_with_backslash() { + let json: Json = r"path\to\file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path\\to\\file""#); + } + + #[test] + fn test_string_with_slash() { + let json: Json = "path/to/file".into(); + assert_snapshot!(marshal_to_string(&json), @r#""path/to/file""#); + } + + #[test] + fn test_string_with_newline() { + let json: Json = "line1\nline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\nline2""#); + } + + #[test] + fn test_string_with_carriage_return() { + let json: Json = "line1\rline2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""line1\rline2""#); + } + + #[test] + fn test_string_with_tab() { + let json: Json = "col1\tcol2".into(); + assert_snapshot!(marshal_to_string(&json), @r#""col1\tcol2""#); + } + + #[test] + fn test_string_with_backspace() { + let json: Json = "text\x08back".into(); + assert_snapshot!(marshal_to_string(&json), @r#""text\bback""#); + } + + #[test] + fn test_string_with_form_feed() { + let json: Json = "page\x0Cnew".into(); + assert_snapshot!(marshal_to_string(&json), @r#""page\fnew""#); + } + + #[test] + fn test_string_with_control_chars() { + let json: Json = "test\x01\x02\x03".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test\u0001\u0002\u0003""#); + } + + #[test] + fn test_string_with_all_escapes() { + let json: Json = "\"\\/\n\r\t\x08\x0C".into(); + assert_snapshot!(marshal_to_string(&json), @r#""\"\\/\n\r\t\b\f""#); + } + + #[test] + fn test_array_empty() { + let json: Json = Vec::<i32>::new().into(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_array_single_element() { + let json: Json = vec![42].into(); + assert_snapshot!(marshal_to_string(&json), @"[42]"); + } + + #[test] + fn test_array_multiple_elements() { + let json: Json = vec![1, 2, 3].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_array_mixed_types() { + let json = Json::Array(vec![ + Json::Null, + true.into(), + 42.into(), + 3.134.try_into().unwrap(), + "hello".into(), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"[null, true, 42, 3.134, "hello"]"#); + } + + #[test] + fn test_array_nested() { + let json = Json::Array(vec![1.into(), vec![2, 3].into(), 4.into()]); + assert_snapshot!(marshal_to_string(&json), @"[1, [2, 3], 4]"); + } + + #[test] + fn test_object_empty() { + let json = Json::Object(vec![]); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_single_field() { + let json = Json::Object(vec![("key".to_string(), "value".into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_multiple_fields() { + let json = Json::Object(vec![ + ("name".to_string(), "Alice".into()), + ("age".to_string(), 30.into()), + ("active".to_string(), true.into()), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_with_escaped_key() { + let json = Json::Object(vec![("key\nwith\nnewlines".to_string(), 42.into())]); + assert_snapshot!(marshal_to_string(&json), @r#"{"key\nwith\nnewlines":42}"#); + } + + #[test] + fn test_object_nested() { + let inner = Json::Object(vec![("inner_key".to_string(), "inner_value".into())]); + let json = Json::Object(vec![("outer_key".to_string(), inner)]); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_from_str() { + let json: Json = "test string".into(); + assert_snapshot!(marshal_to_string(&json), @r#""test string""#); + } + + #[test] + fn test_from_i32() { + let json: Json = 42i32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_i64() { + let json: Json = 9223372036854775807i64.into(); + assert_snapshot!(marshal_to_string(&json), @"9223372036854775807"); + } + + #[test] + fn test_from_u32() { + let json: Json = 42u32.into(); + assert_snapshot!(marshal_to_string(&json), @"42"); + } + + #[test] + fn test_from_u64() { + let json: Json = 18446744073709551615u64.into(); + assert_snapshot!(marshal_to_string(&json), @"18446744073709551615"); + } + + #[test] + fn test_unsigned_integer_zero() { + let json: Json = 0u64.into(); + assert_snapshot!(marshal_to_string(&json), @"0"); + } + + #[test] + fn test_from_bool() { + let json_true: Json = true.into(); + let json_false: Json = false.into(); + assert_snapshot!(marshal_to_string(&json_true), @"true"); + assert_snapshot!(marshal_to_string(&json_false), @"false"); + } + + #[test] + fn test_from_vec() { + let json: Json = vec![1i32, 2i32, 3i32].into(); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_from_vec_strings() { + let json: Json = vec!["a", "b", "c"].into(); + assert_snapshot!(marshal_to_string(&json), @r#"["a", "b", "c"]"#); + } + + #[test] + fn test_complex_nested_structure() { + let settings = Json::Object(vec![ + ("notifications".to_string(), true.into()), + ("theme".to_string(), "dark".into()), + ]); + + let json = Json::Object(vec![ + ("id".to_string(), 1.into()), + ("name".to_string(), "Alice".into()), + ("tags".to_string(), vec!["admin", "user"].into()), + ("settings".to_string(), settings), + ]); + assert_snapshot!(marshal_to_string(&json), @r#"{"id":1, "name":"Alice", "tags":["admin", "user"], "settings":{"notifications":true, "theme":"dark"}}"#); + } + + #[test] + fn test_deeply_nested_arrays() { + let json = Json::Array(vec![ + Json::Array(vec![vec![1, 2].into(), 3.into()]), + 4.into(), + ]); + assert_snapshot!(marshal_to_string(&json), @"[[[1, 2], 3], 4]"); + } + + #[test] + fn test_unicode_string() { + let json: Json = "兵马俑".into(); + assert_snapshot!(marshal_to_string(&json), @r#""兵马俑""#); + } + + #[test] + fn test_json_array_convenience() { + let json = Json::array(vec![1, 2, 3]); + assert_snapshot!(marshal_to_string(&json), @"[1, 2, 3]"); + } + + #[test] + fn test_json_array_from_iterator() { + let json = Json::array([1, 2, 3].iter().map(|&x| x * 2)); + assert_snapshot!(marshal_to_string(&json), @"[2, 4, 6]"); + } + + #[test] + fn test_json_empty_array() { + let json = Json::empty_array(); + assert_snapshot!(marshal_to_string(&json), @"[]"); + } + + #[test] + fn test_object_builder_empty() { + let json = Json::object().build(); + assert_snapshot!(marshal_to_string(&json), @"{}"); + } + + #[test] + fn test_object_builder_single_field() { + let json = Json::object().insert("key", "value").build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"key":"value"}"#); + } + + #[test] + fn test_object_builder_multiple_fields() { + let json = Json::object() + .insert("name", "Alice") + .insert("age", 30) + .insert("active", true) + .build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"name":"Alice", "age":30, "active":true}"#); + } + + #[test] + fn test_object_builder_with_nested_objects() { + let inner = Json::object().insert("inner_key", "inner_value").build(); + let json = Json::object().insert("outer_key", inner).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"outer_key":{"inner_key":"inner_value"}}"#); + } + + #[test] + fn test_object_builder_with_array() { + let json = Json::object().insert("items", vec![1, 2, 3]).build(); + assert_snapshot!(marshal_to_string(&json), @r#"{"items":[1, 2, 3]}"#); + } + + #[test] + fn test_display_trait() { + let json = Json::object() + .insert("name", "Bob") + .insert("count", 42) + .build(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#"{"name":"Bob", "count":42}"#); + } + + #[test] + fn test_display_trait_array() { + let json: Json = vec![1, 2, 3].into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @"[1, 2, 3]"); + } + + #[test] + fn test_display_trait_string() { + let json: Json = "test".into(); + let display_output = format!("{}", json); + assert_snapshot!(display_output, @r#""test""#); + } + + #[test] + fn test_from_usize() { + let json: Json = 123usize.into(); + assert_snapshot!(marshal_to_string(&json), @"123"); + } + + #[test] + fn test_from_usize_large() { + let json: Json = usize::MAX.into(); + let expected = format!("{}", usize::MAX); + assert_eq!(marshal_to_string(&json), expected); + } + + #[test] + fn test_json_error_float_display() { + let err = JsonError::FloatError(f64::NAN); + let display_output = format!("{}", err); + assert!(display_output.contains("Cannot serialize float")); + assert!(display_output.contains("NaN")); + } + + #[test] + fn test_json_error_float_display_infinity() { + let err = JsonError::FloatError(f64::INFINITY); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"Cannot serialize float inf"); + } + + #[test] + fn test_json_error_io_display() { + let io_err = io::Error::new(io::ErrorKind::WriteZero, "write error"); + let err = JsonError::IoError(io_err); + let display_output = format!("{}", err); + assert_snapshot!(display_output, @"write error"); + } + + #[test] + fn test_io_error_during_marshal() { + struct FailingWriter; + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> io::Result<usize> { + Err(io::Error::other("simulated write failure")) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let json: Json = "test".into(); + let mut writer = FailingWriter; + let result = json.marshal(&mut writer); + assert!(result.is_err()); + assert!(matches!(result, Err(JsonError::IoError(_)))); + } + + #[test] + fn test_clone_json() { + let json1: Json = vec![1, 2, 3].into(); + let json2 = json1.clone(); + assert_eq!(json1, json2); + } + + #[test] + fn test_debug_json() { + let json: Json = "test".into(); + let debug_output = format!("{:?}", json); + assert!(debug_output.contains("String")); + assert!(debug_output.contains("test")); + } + + #[test] + fn test_partial_eq_json() { + let json1: Json = 42.into(); + let json2: Json = 42.into(); + let json3: Json = 43.into(); + assert_eq!(json1, json2); + assert_ne!(json1, json3); + } +} diff --git a/zjit/src/lib.rs b/zjit/src/lib.rs index b36bf6515e..1440b6ff69 100644 --- a/zjit/src/lib.rs +++ b/zjit/src/lib.rs @@ -1,6 +1,10 @@ #![allow(dead_code)] #![allow(static_mut_refs)] +#![allow(clippy::enum_variant_names)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::needless_bool)] + // Add std docs to cargo doc. #[doc(inline)] pub use std; @@ -11,6 +15,7 @@ mod cruby; mod cruby_methods; mod hir; mod hir_type; +mod hir_effect; mod codegen; mod stats; mod cast; @@ -22,7 +27,20 @@ mod disasm; mod options; mod profile; mod invariants; -#[cfg(test)] -mod assertions; mod bitset; mod gc; +mod jit_frame; +mod payload; +mod json; +mod ttycolors; + +/// Pull in YJIT's symbols for linking the test binary in `make zjit-test`. The test binary builds +/// ZJIT symbols and they should take precendence over the ones built for miniruby, so libminiruby +/// doesn't include any ZJIT code. But, in removing from libminiruby the object which contains all +/// rust code, including ZJIT code, we also remove all YJIT symbols which the rest of libminiruby +/// might request in YJIT+ZJIT configurations. We add back the YJIT symbols here. +/// +/// Only relevant for YJIT+ZJIT configurations, but building YJIT is fast, so always do it for the +/// test binary for simplicity. +#[cfg(test)] +use yjit as _; diff --git a/zjit/src/options.rs b/zjit/src/options.rs index 94a6988a4f..5ddaee1951 100644 --- a/zjit/src/options.rs +++ b/zjit/src/options.rs @@ -1,20 +1,41 @@ -use std::{ffi::{CStr, CString}, ptr::null}; +//! Configurable options for ZJIT. + +use std::{ffi::{CStr, CString}, fs::File, ptr::null}; use std::os::raw::{c_char, c_int, c_uint}; use crate::cruby::*; +use crate::stats::Counter; use std::collections::HashSet; +/// Type of symbols to dump into /tmp/perf-{pid}.map +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PerfMap { + /// Dump one symbol per ISEQ + ISEQ, + /// Dump one symbol per HIR instruction + HIR, +} + +/// Default --zjit-num-profiles +const DEFAULT_NUM_PROFILES: NumProfiles = 5; +pub type NumProfiles = u16; + +/// Default --zjit-call-threshold. This should be large enough to avoid compiling +/// warmup code, but small enough to perform well on micro-benchmarks. +pub const DEFAULT_CALL_THRESHOLD: CallThreshold = 30; +pub type CallThreshold = u64; + /// Number of calls to start profiling YARV instructions. /// They are profiled `rb_zjit_call_threshold - rb_zjit_profile_threshold` times, /// which is equal to --zjit-num-profiles. #[unsafe(no_mangle)] #[allow(non_upper_case_globals)] -pub static mut rb_zjit_profile_threshold: u64 = 1; +pub static mut rb_zjit_profile_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD - DEFAULT_NUM_PROFILES as CallThreshold; /// Number of calls to compile ISEQ with ZJIT at jit_compile() in vm.c. /// --zjit-call-threshold=1 compiles on first execution without profiling information. #[unsafe(no_mangle)] #[allow(non_upper_case_globals)] -pub static mut rb_zjit_call_threshold: u64 = 2; +pub static mut rb_zjit_call_threshold: CallThreshold = DEFAULT_CALL_THRESHOLD; /// ZJIT command-line options. This is set before rb_zjit_init() sets /// ZJITState so that we can query some options while loading builtins. @@ -26,15 +47,29 @@ pub struct Options { /// Note that the command line argument is expressed in MiB and not bytes. pub exec_mem_bytes: usize, + /// Hard limit of ZJIT's total memory usage. + /// Note that the command line argument is expressed in MiB and not bytes. + pub mem_bytes: usize, + /// Number of times YARV instructions should be profiled. - pub num_profiles: u8, + pub num_profiles: NumProfiles, - /// Enable YJIT statsitics + /// Enable ZJIT statistics pub stats: bool, + /// Print stats on exit (when stats is also true) + pub print_stats: bool, + + /// Print stats to file on exit (when stats is also true) + pub print_stats_file: Option<std::path::PathBuf>, + /// Enable debug logging pub debug: bool, + // Whether to enable JIT at boot. This option prevents other + // ZJIT tuning options from enabling ZJIT at boot. + pub disable: bool, + /// Turn off the HIR optimizer pub disable_hir_opt: bool, @@ -44,40 +79,69 @@ pub struct Options { /// Dump High-level IR after optimization, right before codegen. pub dump_hir_opt: Option<DumpHIR>, - pub dump_hir_graphviz: bool, + /// Dump High-level IR to the given file in Graphviz format after optimization + pub dump_hir_graphviz: Option<std::path::PathBuf>, + + /// Dump High-level IR in Iongraph JSON format after optimization to /tmp/zjit-iongraph-{$PID} + pub dump_hir_iongraph: bool, /// Dump low-level IR - pub dump_lir: bool, + pub dump_lir: Option<HashSet<DumpLIR>>, /// Dump all compiled machine code. - pub dump_disasm: bool, + pub dump_disasm: Option<DumpDisasm>, + + /// Trace and write side exit source maps to /tmp for stackprof. + pub trace_side_exits: Option<TraceExits>, + + /// Frequency of tracing side exits. + pub trace_side_exits_sample_interval: usize, + + /// Trace compilation phases as Perfetto duration events. + pub trace_compiles: bool, + + /// Trace invalidation events as Perfetto duration events. + pub trace_invalidation: bool, /// Dump code map to /tmp for performance profilers. - pub perf: bool, + pub perf: Option<PerfMap>, /// List of ISEQs that can be compiled, identified by their iseq_get_location() pub allowed_iseqs: Option<HashSet<String>>, /// Path to a file where compiled ISEQs will be saved. - pub log_compiled_iseqs: Option<String>, + pub log_compiled_iseqs: Option<std::path::PathBuf>, + + /// Maximum number of versions per ISEQ + pub max_versions: usize, } impl Default for Options { fn default() -> Self { Options { exec_mem_bytes: 64 * 1024 * 1024, - num_profiles: 1, + mem_bytes: 128 * 1024 * 1024, + num_profiles: DEFAULT_NUM_PROFILES, stats: false, + print_stats: false, + print_stats_file: None, debug: false, + disable: false, disable_hir_opt: false, dump_hir_init: None, dump_hir_opt: None, - dump_hir_graphviz: false, - dump_lir: false, - dump_disasm: false, - perf: false, + dump_hir_graphviz: None, + dump_hir_iongraph: false, + dump_lir: None, + dump_disasm: None, + trace_side_exits: None, + trace_side_exits_sample_interval: 0, + trace_compiles: false, + trace_invalidation: false, + perf: None, allowed_iseqs: None, log_compiled_iseqs: None, + max_versions: 2, } } } @@ -85,20 +149,41 @@ impl Default for Options { /// `ruby --help` descriptions for user-facing options. Do not add options for ZJIT developers. /// Note that --help allows only 80 chars per line, including indentation, and it also puts the /// description in a separate line if the option name is too long. 80-char limit --> | (any character beyond this `|` column fails the test) -pub const ZJIT_OPTIONS: &'static [(&str, &str)] = &[ - // TODO: Hide --zjit-exec-mem-size from ZJIT_OPTIONS once we add --zjit-mem-size (Shopify/ruby#686) - ("--zjit-exec-mem-size=num", - "Size of executable memory block in MiB (default: 64)."), +pub const ZJIT_OPTIONS: &[(&str, &str)] = &[ + ("--zjit-mem-size=num", + "Max amount of memory that ZJIT can use in MiB (default: 128)."), ("--zjit-call-threshold=num", - "Number of calls to trigger JIT (default: 2)."), + "Number of calls to trigger JIT (default: 30)."), ("--zjit-num-profiles=num", - "Number of profiled calls before JIT (default: 1, max: 255)."), - ("--zjit-stats", "Enable collecting ZJIT statistics."), - ("--zjit-perf", "Dump ISEQ symbols into /tmp/perf-{}.map for Linux perf."), + "Number of profiled calls before JIT (default: 5)."), + ("--zjit-stats-quiet", + "Collect ZJIT stats and suppress output."), + ("--zjit-stats[=file]", + "Collect ZJIT stats (=file to write to a file)."), + ("--zjit-disable", + "Disable ZJIT for lazily enabling it with RubyVM::ZJIT.enable."), + ("--zjit-perf[=iseq|hir]", + "Dump symbols for Linux perf /tmp/perf-{}.map (default: iseq)."), ("--zjit-log-compiled-iseqs=path", "Log compiled ISEQs to the file. The file will be truncated."), + ("--zjit-trace-exits[=counter]", + "Record source on side-exit. `Counter` picks specific counter."), + ("--zjit-trace-exits-sample-rate=num", + "Frequency at which to record side exits. Must be `usize`."), + ("--zjit-trace-compiles", + "Record compilation phases as Perfetto trace events."), + ("--zjit-trace-invalidation", + "Record invalidation events as Perfetto trace events."), ]; +#[derive(Copy, Clone, Debug)] +pub enum TraceExits { + // Trace all exits + All, + // Trace exits for a specific `Counter` + Counter(Counter), +} + #[derive(Clone, Copy, Debug)] pub enum DumpHIR { // Dump High-level IR without Snapshot @@ -109,6 +194,59 @@ pub enum DumpHIR { Debug, } +/// --zjit-dump-lir values. Using snake_case to stringify the exact filter value. +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub enum DumpLIR { + /// Dump the initial LIR + init, + /// Dump LIR after {arch}_split + split, + /// Dump LIR after alloc_regs + alloc_regs, + /// Dump LIR after compile_exits + compile_exits, + /// Dump LIR after resolve_parallel_mov + resolve_parallel_mov, + /// Dump LIR after {arch}_scratch_split + scratch_split, + /// Dump live intervals grid before alloc_regs + live_intervals, +} + +#[derive(Clone, Copy, Debug)] +pub enum DumpDisasm { + Stdout, + File(std::os::unix::io::RawFd), +} + +/// All compiler stages for --zjit-dump-lir=all. +const DUMP_LIR_ALL: &[DumpLIR] = &[ + DumpLIR::init, + DumpLIR::split, + DumpLIR::alloc_regs, + DumpLIR::compile_exits, + DumpLIR::resolve_parallel_mov, + DumpLIR::scratch_split, + DumpLIR::live_intervals, +]; + +/// Maximum value for --zjit-mem-size/--zjit-exec-mem-size in MiB. +/// We set 1TiB just to avoid overflow. We could make it smaller. +const MAX_MEM_MIB: usize = 1024 * 1024; + +/// Macro to dump LIR if --zjit-dump-lir is specified +macro_rules! asm_dump { + ($asm:expr, $target:ident) => { + if let Some(crate::options::Options { dump_lir: Some(dump_lirs), .. }) = unsafe { crate::options::OPTIONS.as_ref() } { + if dump_lirs.contains(&crate::options::DumpLIR::$target) { + println!("LIR {}:\n{}", stringify!($target), $asm); + } + } + }; +} +pub(crate) use asm_dump; + /// Macro to get an option value by name macro_rules! get_option { // Unsafe is ok here because options are initialized @@ -119,6 +257,14 @@ macro_rules! get_option { } pub(crate) use get_option; +/// Macro to reference an option value by name. +macro_rules! get_option_ref { + ($option_name:ident) => { + unsafe { crate::options::OPTIONS.as_ref() }.unwrap().$option_name.as_ref() + }; +} +pub(crate) use get_option_ref; + /// Set default values to ZJIT options. Setting Some to OPTIONS will make `#with_jit` /// enable the JIT hook while not enabling compilation yet. #[unsafe(no_mangle)] @@ -147,11 +293,11 @@ fn parse_jit_list(path_like: &str) -> HashSet<String> { } } } else { - eprintln!("Failed to read JIT list from '{}'", path_like); + eprintln!("Failed to read JIT list from '{path_like}'"); } eprintln!("JIT list:"); for item in &result { - eprintln!(" {}", item); + eprintln!(" {item}"); } result } @@ -179,17 +325,19 @@ fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { ("", "") => {}, // Simply --zjit ("mem-size", _) => match opt_val.parse::<usize>() { - Ok(n) => { - // Reject 0 or too large values that could overflow. - // The upper bound is 1 TiB but we could make it smaller. - if n == 0 || n > 1024 * 1024 { - return None - } + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { + // Convert from MiB to bytes internally for convenience + options.mem_bytes = n * 1024 * 1024; + } + _ => return None, + }, + ("exec-mem-size", _) => match opt_val.parse::<usize>() { + Ok(n) if (1..=MAX_MEM_MIB).contains(&n) => { // Convert from MiB to bytes internally for convenience options.exec_mem_bytes = n * 1024 * 1024; } - Err(_) => return None, + _ => return None, }, ("call-threshold", _) => match opt_val.parse() { @@ -208,41 +356,151 @@ fn parse_option(str_ptr: *const std::os::raw::c_char) -> Option<()> { Err(_) => return None, }, + ("max-versions", _) => match opt_val.parse() { + Ok(n) => options.max_versions = n, + Err(_) => return None, + }, + + ("stats-quiet", _) => { + options.stats = true; + options.print_stats = false; + } + ("stats", "") => { options.stats = true; + options.print_stats = true; + } + ("stats", path) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path) + .map_err(|e| eprintln!("Failed to open file '{}': {}", path, e)) + .ok(); + let canonical_path = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.stats = true; + options.print_stats_file = Some(canonical_path); } + ("trace-exits", exits) => { + options.trace_side_exits = match exits { + "" => Some(TraceExits::All), + name => Some(Counter::get(name).map(TraceExits::Counter)?), + } + } + + ("trace-exits-sample-rate", sample_interval) => { + // If not already set, then set it to `TraceExits::All` by default. + if options.trace_side_exits.is_none() { + options.trace_side_exits = Some(TraceExits::All); + } + // `sample_interval ` must provide a string that can be validly parsed to a `usize`. + options.trace_side_exits_sample_interval = sample_interval.parse::<usize>().ok()?; + } + + ("trace-compiles", "") => options.trace_compiles = true, + + ("trace-invalidation", "") => options.trace_invalidation = true, + ("debug", "") => options.debug = true, + ("disable", "") => options.disable = true, + ("disable-hir-opt", "") => options.disable_hir_opt = true, // --zjit-dump-hir dumps the actual input to the codegen, which is currently the same as --zjit-dump-hir-opt. ("dump-hir" | "dump-hir-opt", "") => options.dump_hir_opt = Some(DumpHIR::WithoutSnapshot), ("dump-hir" | "dump-hir-opt", "all") => options.dump_hir_opt = Some(DumpHIR::All), ("dump-hir" | "dump-hir-opt", "debug") => options.dump_hir_opt = Some(DumpHIR::Debug), - ("dump-hir-graphviz", "") => options.dump_hir_graphviz = true, ("dump-hir-init", "") => options.dump_hir_init = Some(DumpHIR::WithoutSnapshot), ("dump-hir-init", "all") => options.dump_hir_init = Some(DumpHIR::All), ("dump-hir-init", "debug") => options.dump_hir_init = Some(DumpHIR::Debug), - ("dump-lir", "") => options.dump_lir = true, + ("dump-hir-graphviz", "") => options.dump_hir_graphviz = Some("/dev/stderr".into()), + ("dump-hir-graphviz", _) => { + // Truncate the file if it exists + std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(opt_val) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) + .ok(); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.dump_hir_graphviz = Some(opt_val); + } + + ("dump-hir-iongraph", "") => options.dump_hir_iongraph = true, + + ("dump-lir", "") => options.dump_lir = Some(HashSet::from([DumpLIR::init])), + ("dump-lir", filters) => { + let mut dump_lirs = HashSet::new(); + for filter in filters.split(',') { + let dump_lir = match filter { + "all" => { + for &dump_lir in DUMP_LIR_ALL { + dump_lirs.insert(dump_lir); + } + continue; + } + "init" => DumpLIR::init, + "split" => DumpLIR::split, + "alloc_regs" => DumpLIR::alloc_regs, + "compile_exits" => DumpLIR::compile_exits, + "resolve_parallel_mov" => DumpLIR::resolve_parallel_mov, + "scratch_split" => DumpLIR::scratch_split, + "live_intervals" => DumpLIR::live_intervals, + _ => { + let valid_options = DUMP_LIR_ALL.iter().map(|opt| format!("{opt:?}")).collect::<Vec<_>>().join(", "); + eprintln!("invalid --zjit-dump-lir option: '{filter}'"); + eprintln!("valid --zjit-dump-lir options: all, {valid_options}"); + return None; + } + }; + dump_lirs.insert(dump_lir); + } + options.dump_lir = Some(dump_lirs); + } + + ("dump-disasm", _) => { + if !cfg!(feature = "disasm") { + eprintln!("WARNING: the {opt_name} option works best when ZJIT is built in dev mode, i.e. ./configure --enable-zjit=dev"); + } - ("dump-disasm", "") => options.dump_disasm = true, + match opt_val { + "" => options.dump_disasm = Some(DumpDisasm::Stdout), + directory => { + let path = format!("{directory}/zjit_{}.log", std::process::id()); + match File::options().create(true).append(true).open(&path) { + Ok(file) => { + use std::os::unix::io::IntoRawFd; + eprintln!("ZJIT disasm dump: {path}"); + options.dump_disasm = Some(DumpDisasm::File(file.into_raw_fd())); + } + Err(err) => eprintln!("Failed to create {path}: {err}"), + } + } + } + } - ("perf", "") => options.perf = true, + ("perf", "" | "iseq") => options.perf = Some(PerfMap::ISEQ), + ("perf", "hir") => options.perf = Some(PerfMap::HIR), - ("allowed-iseqs", _) if opt_val != "" => options.allowed_iseqs = Some(parse_jit_list(opt_val)), - ("log-compiled-iseqs", _) if opt_val != "" => { + ("allowed-iseqs", _) if !opt_val.is_empty() => options.allowed_iseqs = Some(parse_jit_list(opt_val)), + ("log-compiled-iseqs", _) if !opt_val.is_empty() => { // Truncate the file if it exists std::fs::OpenOptions::new() .create(true) .write(true) .truncate(true) .open(opt_val) - .map_err(|e| eprintln!("Failed to open file '{}': {}", opt_val, e)) + .map_err(|e| eprintln!("Failed to open file '{opt_val}': {e}")) .ok(); - options.log_compiled_iseqs = Some(opt_val.into()); + let opt_val = std::fs::canonicalize(opt_val).unwrap_or_else(|_| opt_val.into()); + options.log_compiled_iseqs = Some(opt_val); } _ => return None, // Option name not recognized @@ -259,12 +517,27 @@ fn update_profile_threshold() { unsafe { rb_zjit_profile_threshold = 0; } } else { // Otherwise, profile instructions at least once. - let num_profiles = get_option!(num_profiles) as u64; - unsafe { rb_zjit_profile_threshold = rb_zjit_call_threshold.saturating_sub(num_profiles).max(1) }; + let num_profiles = get_option!(num_profiles); + unsafe { rb_zjit_profile_threshold = rb_zjit_call_threshold.saturating_sub(num_profiles.into()).max(1) }; } } -/// Print YJIT options for `ruby --help`. `width` is width of option parts, and +/// Update --zjit-call-threshold for testing +#[cfg(test)] +pub fn set_call_threshold(call_threshold: CallThreshold) { + unsafe { rb_zjit_call_threshold = call_threshold; } + rb_zjit_prepare_options(); + update_profile_threshold(); +} + +/// Enable --zjit-stats for testing +#[cfg(test)] +pub fn enable_zjit_stats() { + rb_zjit_prepare_options(); + unsafe { OPTIONS.as_mut() }.unwrap().stats = true; +} + +/// Print ZJIT options for `ruby --help`. `width` is width of option parts, and /// `columns` is indent width of descriptions. #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_show_usage(help: c_int, highlight: c_int, width: c_uint, columns: c_int) { @@ -289,15 +562,13 @@ macro_rules! debug { } pub(crate) use debug; -/// Return Qtrue if --zjit* has been specified. For the `#with_jit` hook, -/// this becomes Qtrue before ZJIT is actually initialized and enabled. +/// Return true if ZJIT should be enabled at boot. #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_option_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { - // If any --zjit* option is specified, OPTIONS becomes Some. - if unsafe { OPTIONS.is_some() } { - Qtrue +pub extern "C" fn rb_zjit_option_enable() -> bool { + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| !opts.disable) { + true } else { - Qfalse + false } } @@ -305,9 +576,56 @@ pub extern "C" fn rb_zjit_option_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_stats_enabled_p(_ec: EcPtr, _self: VALUE) -> VALUE { // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. - if unsafe { OPTIONS.as_ref() }.map_or(false, |opts| opts.stats) { + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats) { + Qtrue + } else { + Qfalse + } +} + +/// Return Qtrue if stats should be printed at exit. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_print_stats_p(_ec: EcPtr, _self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.stats && opts.print_stats) { Qtrue } else { Qfalse } } + +/// Return path if stats should be printed at exit to a specified file, else Qnil. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_get_stats_file_path_p(_ec: EcPtr, _self: VALUE) -> VALUE { + if let Some(opts) = unsafe { OPTIONS.as_ref() } { + if let Some(ref path) = opts.print_stats_file { + return rust_str_to_ruby(path.as_os_str().to_str().unwrap()); + } + } + Qnil +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_dump_disasm_path() { + unsafe { OPTIONS = Some(Options::default()); } + + let dir = std::env::temp_dir(); + let expected_path = dir.join(format!("zjit_{}.log", std::process::id())); + let option = CString::new(format!("dump-disasm={}", dir.display())).unwrap(); + + assert!(parse_option(option.as_ptr()).is_some()); + + let options = unsafe { OPTIONS.as_ref() }.unwrap(); + match options.dump_disasm { + Some(DumpDisasm::File(fd)) => assert!(fd >= 0), + _ => panic!("expected dump-disasm file output"), + } + assert!(expected_path.exists()); + + let _ = std::fs::remove_file(expected_path); + } +} diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs new file mode 100644 index 0000000000..51b6f4721b --- /dev/null +++ b/zjit/src/payload.rs @@ -0,0 +1,144 @@ +use std::ffi::c_void; +use std::ptr::NonNull; +use crate::codegen::IseqCallRef; +use crate::stats::CompileError; +use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; +use crate::options::get_option; + +pub use crate::jit_frame::JITFrame; + +/// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. +#[derive(Debug)] +pub struct IseqPayload { + /// Type information of YARV instruction operands + pub profile: IseqProfile, + /// JIT code versions. Different versions should have different assumptions. + pub versions: Vec<IseqVersionRef>, + /// Whether a previous compilation of this ISEQ was invalidated due to + /// singleton class creation (violation of [`crate::hir::Invariant::NoSingletonClass`]). + pub was_invalidated_for_singleton_class_creation: bool, + /// Whether `self` is guaranteed to be a heap (non-immediate) object for this + /// ISEQ. Set at compile triggers (entry point / function stub hit) where the + /// owning class is known via the method entry, and consumed in `iseq_to_hir` + /// to type the `self`-producing instructions (`LoadSelf` / `SelfParam` + /// `LoadArg`) as `HeapBasicObject`. Defaults to `false` (the conservative + /// `BasicObject`) when the owner is unknown. + /// See [`crate::cruby::iseq_self_is_heap_object`]. + pub self_is_heap_object: bool, +} + +impl IseqPayload { + fn new() -> Self { + Self { + profile: IseqProfile::new(), + versions: vec![], + was_invalidated_for_singleton_class_creation: false, + self_is_heap_object: false, + } + } + + /// Profile counts are used for compilation policy. + /// When we deoptimize a method that can be recompiled, we need to update the count to collect more profiles. + /// Otherwise, we will generate the same code that was just deoptimized. + pub fn reset_profiles_remaining(&mut self, insn_idx: YarvInsnIdx) { + let num_profiles = get_option!(num_profiles); + self.profile.entry_mut(insn_idx).set_profiles_remaining(num_profiles); + } +} + +/// JIT code version. When the same ISEQ is compiled with a different assumption, a new version is created. +#[derive(Debug)] +pub struct IseqVersion { + /// ISEQ pointer. Stored here to minimize the size of PatchPoint. + pub iseq: IseqPtr, + + /// Compilation status of the ISEQ. It has the JIT code address of the first block if Compiled. + pub status: IseqStatus, + + /// GC offsets of the JIT code. These are the addresses of objects that need to be marked. + pub gc_offsets: Vec<CodePtr>, + + /// JIT-to-JIT calls from the ISEQ. The IseqPayload's ISEQ is the caller of it. + pub outgoing: Vec<IseqCallRef>, + + /// JIT-to-JIT calls to the ISEQ. The IseqPayload's ISEQ is the callee of it. + pub incoming: Vec<IseqCallRef>, +} + +/// We use a raw pointer instead of Rc to save space for refcount +pub type IseqVersionRef = NonNull<IseqVersion>; + +impl IseqVersion { + /// Check if this version was invalidated + pub fn is_invalidated(&self) -> bool { + self.status == IseqStatus::Invalidated + } + + /// Allocate a new IseqVersion to be compiled + pub fn new(iseq: IseqPtr) -> IseqVersionRef { + let version = Self { + iseq, + status: IseqStatus::NotCompiled, + gc_offsets: vec![], + outgoing: vec![], + incoming: vec![], + }; + let version_ptr = Box::into_raw(Box::new(version)); + NonNull::new(version_ptr).expect("no null from Box") + } +} + +/// Set of CodePtrs for an ISEQ +#[derive(Clone, Debug, PartialEq)] +pub struct IseqCodePtrs { + /// Entry for the interpreter + pub start_ptr: CodePtr, + /// Entries for JIT-to-JIT calls + pub jit_entry_ptrs: Vec<CodePtr>, +} + +#[derive(Debug, PartialEq)] +pub enum IseqStatus { + Compiled(IseqCodePtrs), + CantCompile(CompileError), + NotCompiled, + Invalidated, +} + +/// Get a pointer to the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload_ptr(iseq: IseqPtr) -> *mut IseqPayload { + type VoidPtr = *mut c_void; + + unsafe { + let payload = rb_iseq_get_zjit_payload(iseq); + if payload.is_null() { + // Allocate a new payload with Box and transfer ownership to the GC. + // We drop the payload with Box::from_raw when the GC frees the ISEQ and calls us. + // NOTE(alan): Sometimes we read from an ISEQ without ever writing to it. + // We allocate in those cases anyways. + let new_payload = IseqPayload::new(); + let new_payload = Box::into_raw(Box::new(new_payload)); + rb_iseq_set_zjit_payload(iseq, new_payload as VoidPtr); + + new_payload + } else { + payload as *mut IseqPayload + } + } +} + +/// Get the payload object associated with an ISEQ. Create one if none exists. +pub fn get_or_create_iseq_payload(iseq: IseqPtr) -> &'static mut IseqPayload { + let payload_non_null = get_or_create_iseq_payload_ptr(iseq); + payload_ptr_as_mut(payload_non_null) +} + +/// Convert an IseqPayload pointer to a mutable reference. Only one reference +/// should be kept at a time. +pub fn payload_ptr_as_mut(payload_ptr: *mut IseqPayload) -> &'static mut IseqPayload { + // SAFETY: we should have the VM lock and all other Ruby threads should be asleep. So we have + // exclusive mutable access. + // Hmm, nothing seems to stop calling this on the same + // iseq twice, though, which violates aliasing rules. + unsafe { payload_ptr.as_mut() }.unwrap() +} diff --git a/zjit/src/profile.rs b/zjit/src/profile.rs index 771d90cb0e..56a0f9bc3d 100644 --- a/zjit/src/profile.rs +++ b/zjit/src/profile.rs @@ -1,7 +1,10 @@ +//! Profiler for runtime information. + // We use the YARV bytecode constants which have a CRuby-style name #![allow(non_upper_case_globals)] -use crate::{cruby::*, gc::get_or_create_iseq_payload, options::get_option}; +use std::collections::HashMap; +use crate::{cruby::*, payload::get_or_create_iseq_payload, options::{get_option, NumProfiles}}; use crate::distribution::{Distribution, DistributionSummary}; use crate::stats::Counter::profile_time_ns; use crate::stats::with_time_stat; @@ -10,7 +13,7 @@ use crate::stats::with_time_stat; struct Profiler { cfp: CfpPtr, iseq: IseqPtr, - insn_idx: usize, + insn_idx: YarvInsnIdx, } impl Profiler { @@ -37,6 +40,14 @@ impl Profiler { *(sp.offset(-1 - n)) } } + + fn peek_at_self(&self) -> VALUE { + unsafe { rb_get_cfp_self(self.cfp) } + } + + fn peek_at_block_handler(&self) -> VALUE { + unsafe { rb_vm_get_untagged_block_handler(self.cfp) } + } } /// API called from zjit_* instruction. opcode is the bare (non-zjit_*) instruction. @@ -66,22 +77,49 @@ fn profile_insn(bare_opcode: ruby_vminsn_type, ec: EcPtr) { YARVINSN_opt_ge => profile_operands(profiler, profile, 2), YARVINSN_opt_and => profile_operands(profiler, profile, 2), YARVINSN_opt_or => profile_operands(profiler, profile, 2), - YARVINSN_opt_send_without_block => { + YARVINSN_opt_empty_p => profile_operands(profiler, profile, 1), + YARVINSN_opt_aref => profile_operands(profiler, profile, 2), + YARVINSN_opt_ltlt => profile_operands(profiler, profile, 2), + YARVINSN_opt_aset => profile_operands(profiler, profile, 3), + YARVINSN_opt_not => profile_operands(profiler, profile, 1), + YARVINSN_getinstancevariable => profile_self(profiler, profile), + YARVINSN_setinstancevariable => profile_self(profiler, profile), + YARVINSN_definedivar => profile_self(profiler, profile), + YARVINSN_opt_regexpmatch2 => profile_operands(profiler, profile, 2), + YARVINSN_objtostring => profile_operands(profiler, profile, 1), + YARVINSN_opt_length => profile_operands(profiler, profile, 1), + YARVINSN_opt_size => profile_operands(profiler, profile, 1), + YARVINSN_opt_succ => profile_operands(profiler, profile, 1), + YARVINSN_invokeblock => profile_block_handler(profiler, profile), + YARVINSN_getblockparamproxy => profile_getblockparamproxy(profiler, profile), + YARVINSN_invokesuper => profile_invokesuper(profiler, profile), + YARVINSN_opt_send_without_block | YARVINSN_send => { let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); - let argc = unsafe { vm_ci_argc((*cd).ci) }; + let argc = num_arguments_on_stack(cd); // Profile all the arguments and self (+1). - profile_operands(profiler, profile, (argc + 1) as usize); + profile_operands(profiler, profile, argc + 1); } + YARVINSN_splatkw => profile_operands(profiler, profile, 2), _ => {} } - // Once we profile the instruction num_profiles times, we stop profiling it. - profile.num_profiles[profiler.insn_idx] = profile.num_profiles[profiler.insn_idx].saturating_add(1); - if profile.num_profiles[profiler.insn_idx] == get_option!(num_profiles) { + // Once we profile the instruction enough times, we stop profiling it. + let entry = profile.entry_mut(profiler.insn_idx); + entry.profiles_remaining = entry.profiles_remaining.saturating_sub(1); + if entry.profiles_remaining == 0 { unsafe { rb_zjit_iseq_insn_set(profiler.iseq, profiler.insn_idx as u32, bare_opcode); } } } +/// Return the argc as stated in the calldata plus: +/// * 1 if there is an explicit blockarg, since that will be passed on the stack +pub fn num_arguments_on_stack(cd: *const rb_call_data) -> usize { + let ci = unsafe { rb_get_call_data_ci(cd) }; + let flags = unsafe { rb_vm_ci_flag(ci) }; + let has_blockarg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; + (unsafe { vm_ci_argc(ci) }) as usize + has_blockarg as usize +} + const DISTRIBUTION_SIZE: usize = 4; pub type TypeDistribution = Distribution<ProfiledType, DISTRIBUTION_SIZE>; @@ -90,36 +128,118 @@ pub type TypeDistributionSummary = DistributionSummary<ProfiledType, DISTRIBUTIO /// Profile the Type of top-`n` stack operands fn profile_operands(profiler: &mut Profiler, profile: &mut IseqProfile, n: usize) { - let types = &mut profile.opnd_types[profiler.insn_idx]; - if types.is_empty() { - types.resize(n, TypeDistribution::new()); + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(n, TypeDistribution::new()); } - for i in 0..n { + + for (i, profile_type) in entry.opnd_types.iter_mut().enumerate() { let obj = profiler.peek_at_stack((n - i - 1) as isize); // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or // drop them or something let ty = ProfiledType::new(obj); - unsafe { rb_gc_writebarrier(profiler.iseq.into(), ty.class()) }; - types[i].observe(ty); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + profile_type.observe(ty); + } +} + +fn profile_self(profiler: &mut Profiler, profile: &mut IseqProfile) { + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); } + let obj = profiler.peek_at_self(); + // TODO(max): Handle GC-hidden classes like Array, Hash, etc and make them look normal or + // drop them or something + let ty = ProfiledType::new(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); +} + +fn profile_block_handler(profiler: &mut Profiler, profile: &mut IseqProfile) { + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); + } + let obj = profiler.peek_at_block_handler(); + let ty = ProfiledType::object(obj); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); +} + +fn profile_getblockparamproxy(profiler: &mut Profiler, profile: &mut IseqProfile) { + let entry = profile.entry_mut(profiler.insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); + } + + let level = profiler.insn_opnd(1).as_u32(); + let ep = unsafe { get_cfp_ep_level(profiler.cfp, level) }; + let block_handler = unsafe { *ep.offset(VM_ENV_DATA_INDEX_SPECVAL as isize) }; + let untagged = unsafe { rb_vm_untag_block_handler(block_handler) }; + + let ty = ProfiledType::object(untagged); + VALUE::from(profiler.iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); +} + +fn profile_invokesuper(profiler: &mut Profiler, profile: &mut IseqProfile) { + let cme = unsafe { rb_vm_frame_method_entry(profiler.cfp) }; + let cme_value = VALUE(cme as usize); // CME is a T_IMEMO, which is a VALUE + + profile.super_cme.entry(profiler.insn_idx) + .or_insert_with(|| TypeDistribution::new()).observe(ProfiledType::object(cme_value)); + + unsafe { rb_gc_writebarrier(profiler.iseq.into(), cme_value) }; + + let cd: *const rb_call_data = profiler.insn_opnd(0).as_ptr(); + let argc = num_arguments_on_stack(cd); + + // Profile all the arguments and self (+1). + profile_operands(profiler, profile, (argc + 1) as usize); } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct Flags(u32); +pub struct Flags(u32); impl Flags { const NONE: u32 = 0; const IS_IMMEDIATE: u32 = 1 << 0; + /// Object is embedded and the ivar index lands within the object + const IS_EMBEDDED: u32 = 1 << 1; + /// Object is a T_OBJECT + const IS_T_OBJECT: u32 = 1 << 2; + /// Object is a struct with embedded fields + const IS_STRUCT_EMBEDDED: u32 = 1 << 3; + /// Set if the ProfiledType is used for profiling specific objects, not just classes/shapes + const IS_OBJECT_PROFILING: u32 = 1 << 4; + /// Class/module fields_obj is embedded (or absent) + const IS_FIELDS_EMBEDDED: u32 = 1 << 5; + /// Object is a T_CLASS + const IS_T_CLASS: u32 = 1 << 6; + /// Object is a T_MODULE + const IS_T_MODULE: u32 = 1 << 7; + /// Object is a T_DATA + const IS_T_DATA: u32 = 1 << 8; pub fn none() -> Self { Self(Self::NONE) } pub fn immediate() -> Self { Self(Self::IS_IMMEDIATE) } pub fn is_immediate(self) -> bool { (self.0 & Self::IS_IMMEDIATE) != 0 } + pub fn is_embedded(self) -> bool { (self.0 & Self::IS_EMBEDDED) != 0 } + pub fn is_t_object(self) -> bool { (self.0 & Self::IS_T_OBJECT) != 0 } + pub fn is_struct_embedded(self) -> bool { (self.0 & Self::IS_STRUCT_EMBEDDED) != 0 } + pub fn is_object_profiling(self) -> bool { (self.0 & Self::IS_OBJECT_PROFILING) != 0 } + pub fn is_fields_embedded(self) -> bool { (self.0 & Self::IS_FIELDS_EMBEDDED) != 0 } + pub fn is_t_class(self) -> bool { (self.0 & Self::IS_T_CLASS) != 0 } + pub fn is_t_module(self) -> bool { (self.0 & Self::IS_T_MODULE) != 0 } + pub fn is_t_data(self) -> bool { (self.0 & Self::IS_T_DATA) != 0 } } /// opt_send_without_block/opt_plus/... should store: /// * the class of the receiver, so we can do method lookup /// * the shape of the receiver, so we can optimize ivar lookup +/// /// with those two, pieces of information, we can also determine when an object is an immediate: /// * Integer + IS_IMMEDIATE == Fixnum /// * Float + IS_IMMEDIATE == Flonum @@ -141,6 +261,14 @@ impl Default for ProfiledType { } impl ProfiledType { + /// Profile the object itself + fn object(obj: VALUE) -> Self { + let mut flags = Flags::none(); + flags.0 |= Flags::IS_OBJECT_PROFILING; + Self { class: obj, shape: INVALID_SHAPE_ID, flags } + } + + /// Profile the class and shape of the given object fn new(obj: VALUE) -> Self { if obj == Qfalse { return Self { class: unsafe { rb_cFalseClass }, @@ -172,7 +300,35 @@ impl ProfiledType { shape: INVALID_SHAPE_ID, flags: Flags::immediate() }; } - Self { class: obj.class_of(), shape: obj.shape_id_of(), flags: Flags::none() } + let mut flags = Flags::none(); + if obj.embedded_p() { + flags.0 |= Flags::IS_EMBEDDED; + } + if obj.struct_embedded_p() { + flags.0 |= Flags::IS_STRUCT_EMBEDDED; + } + if unsafe { RB_TYPE_P(obj, RUBY_T_OBJECT) } { + flags.0 |= Flags::IS_T_OBJECT; + } + if unsafe { RB_TYPE_P(obj, RUBY_T_CLASS) } { + flags.0 |= Flags::IS_T_CLASS; + if obj.class_fields_embedded_p() { + flags.0 |= Flags::IS_FIELDS_EMBEDDED; + } + } + if unsafe { RB_TYPE_P(obj, RUBY_T_MODULE) } { + flags.0 |= Flags::IS_T_MODULE; + if obj.class_fields_embedded_p() { + flags.0 |= Flags::IS_FIELDS_EMBEDDED; + } + } + if obj.data_p() { + flags.0 |= Flags::IS_T_DATA; + if obj.data_fields_embedded_p() { + flags.0 |= Flags::IS_FIELDS_EMBEDDED; + } + } + Self { class: obj.class_of(), shape: obj.shape_id_of(), flags } } pub fn empty() -> Self { @@ -191,10 +347,53 @@ impl ProfiledType { self.shape } + pub fn flags(&self) -> Flags { + self.flags + } + + /// For ivar access, you need to know the index in the fields array (described by the shape) + /// and the way to get the fields array (described by the builtin type). Both pieces of + /// information are on the `RBasic::flags` field. This method returns expected masked flags + /// for guarding. + pub fn rbasic_flags_and_mask(&self) -> (u64, u64) { + let shape_flag_shift = u64::from(RB_SHAPE_FLAG_SHIFT); + let (shape, shape_mask) = (u64::from(self.shape().0) << shape_flag_shift, !0 << shape_flag_shift); + let (builtin_type, type_mask) = if self.flags().is_t_object() { + (RUBY_T_OBJECT, RUBY_T_MASK) + } else if self.flags().is_t_class() { + // Check class first since `Class < Module` + (RUBY_T_CLASS, RUBY_T_MASK) + } else if self.flags().is_t_module() { + (RUBY_T_MODULE, RUBY_T_MASK) + } else if self.flags().is_t_data() { + (RUBY_T_DATA, RUBY_T_MASK) + } else { + (0, 0) + }; + (shape | u64::from(builtin_type), shape_mask | u64::from(type_mask)) + } + pub fn is_fixnum(&self) -> bool { self.class == unsafe { rb_cInteger } && self.flags.is_immediate() } + pub fn is_string(&self) -> bool { + if self.flags.is_object_profiling() { + panic!("should not call is_string on object-profiled ProfiledType"); + } + // Fast paths for immediates and exact-class + if self.flags.is_immediate() { + return false; + } + + let string = unsafe { rb_cString }; + if self.class == string{ + return true; + } + + self.class.is_subclass_of(string) == ClassRelationship::Subclass + } + pub fn is_flonum(&self) -> bool { self.class == unsafe { rb_cFloat } && self.flags.is_immediate() } @@ -216,49 +415,174 @@ impl ProfiledType { } } +/// Per-instruction profile entry, stored sparsely in a sorted Vec. +#[derive(Debug)] +pub struct ProfileEntry { + /// YARV instruction index + insn_idx: u32, + /// Type information of YARV instruction operands + opnd_types: Vec<TypeDistribution>, + /// Number of profiles remaining before recompilation. Counts down from --zjit-num-profiles. + profiles_remaining: NumProfiles, +} + +impl ProfileEntry { + pub fn set_profiles_remaining(&mut self, num_profiles: NumProfiles) { + self.profiles_remaining = num_profiles; + } +} + #[derive(Debug)] pub struct IseqProfile { - /// Type information of YARV instruction operands, indexed by the instruction index - opnd_types: Vec<Vec<TypeDistribution>>, + /// Sparse storage of per-instruction profile data, sorted by instruction index. + /// Only instructions that have actually been profiled have entries here. + entries: Vec<ProfileEntry>, - /// Number of profiled executions for each YARV instruction, indexed by the instruction index - num_profiles: Vec<u8>, + /// Method entries for `super` calls (stored as VALUE to be GC-safe) + super_cme: HashMap<YarvInsnIdx, TypeDistribution> } impl IseqProfile { - pub fn new(iseq_size: u32) -> Self { + pub fn new() -> Self { Self { - opnd_types: vec![vec![]; iseq_size as usize], - num_profiles: vec![0; iseq_size as usize], + entries: Vec::new(), + super_cme: HashMap::new(), + } + } + + /// Get or create a mutable profile entry for the given instruction index. + pub fn entry_mut(&mut self, insn_idx: YarvInsnIdx) -> &mut ProfileEntry { + let idx = insn_idx as u32; + match self.entries.binary_search_by_key(&idx, |e| e.insn_idx) { + Ok(i) => &mut self.entries[i], + Err(i) => { + self.entries.insert(i, ProfileEntry { + insn_idx: idx, + opnd_types: Vec::new(), + profiles_remaining: get_option!(num_profiles), + }); + &mut self.entries[i] + } } } + /// Get a profile entry for the given instruction index (read-only). + fn entry(&self, insn_idx: YarvInsnIdx) -> Option<&ProfileEntry> { + let idx = insn_idx as u32; + self.entries.binary_search_by_key(&idx, |e| e.insn_idx) + .ok().map(|i| &self.entries[i]) + } + + /// Check if enough profiles have been gathered for this instruction. + pub fn done_profiling_at(&self, insn_idx: YarvInsnIdx) -> bool { + self.entry(insn_idx).map_or(false, |e| e.profiles_remaining == 0) + } + + /// Profile send operands from the stack at runtime. + /// `sp` is the current stack pointer (after the args and receiver). + /// `argc` is the number of arguments (not counting receiver). + /// Returns true if enough profiles have been gathered and the ISEQ should be recompiled. + pub fn profile_send_at(&mut self, iseq: IseqPtr, insn_idx: YarvInsnIdx, sp: *const VALUE, argc: usize) -> bool { + let n = argc + 1; // args + receiver + let entry = self.entry_mut(insn_idx); + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(n, TypeDistribution::new()); + } + for i in 0..n { + let obj = unsafe { *sp.offset(i as isize - n as isize) }; + let ty = ProfiledType::new(obj); + VALUE::from(iseq).write_barrier(ty.class()); + entry.opnd_types[i].observe(ty); + } + entry.profiles_remaining = entry.profiles_remaining.saturating_sub(1); + entry.profiles_remaining == 0 + } + + /// Profile self for a shape guard exit at runtime. + /// This may be called on an instruction that was already profiled by YARV, + /// so we reset the counter to re-profile with the new shapes seen at runtime. + /// Returns true if enough profiles have been gathered and the ISEQ should be recompiled. + pub fn profile_self_at(&mut self, iseq: IseqPtr, insn_idx: YarvInsnIdx, self_val: VALUE) -> bool { + let entry = self.entry_mut(insn_idx); + // Reset profiling if the previous round already finished (stale YARV profiles). + // This ensures we collect num_profiles samples of the new shapes before recompiling. + if entry.profiles_remaining == 0 { + entry.profiles_remaining = get_option!(num_profiles); + } + if entry.opnd_types.is_empty() { + entry.opnd_types.resize(1, TypeDistribution::new()); + } + let ty = ProfiledType::new(self_val); + VALUE::from(iseq).write_barrier(ty.class()); + entry.opnd_types[0].observe(ty); + entry.profiles_remaining = entry.profiles_remaining.saturating_sub(1); + entry.profiles_remaining == 0 + } + /// Get profiled operand types for a given instruction index - pub fn get_operand_types(&self, insn_idx: usize) -> Option<&[TypeDistribution]> { - self.opnd_types.get(insn_idx).map(|v| &**v) + pub fn get_operand_types(&self, insn_idx: YarvInsnIdx) -> Option<&[TypeDistribution]> { + self.entry(insn_idx).map(|e| e.opnd_types.as_slice()).filter(|s| !s.is_empty()) + } + + pub fn get_super_method_entry(&self, insn_idx: YarvInsnIdx) -> Option<*const rb_callable_method_entry_t> { + let Some(entry) = self.super_cme.get(&insn_idx) else { return None }; + let summary = TypeDistributionSummary::new(entry); + + if summary.is_monomorphic() { + Some(summary.bucket(0).class.0 as *const rb_callable_method_entry_t) + } else { + None + } } /// Run a given callback with every object in IseqProfile pub fn each_object(&self, callback: impl Fn(VALUE)) { - for operands in &self.opnd_types { - for distribution in operands { + for entry in &self.entries { + for distribution in &entry.opnd_types { for profiled_type in distribution.each_item() { // If the type is a GC object, call the callback callback(profiled_type.class); } } } + + for super_cme_values in self.super_cme.values() { + for profiled_type in super_cme_values.each_item() { + callback(profiled_type.class) + } + } } - /// Run a given callback with a mutable reference to every object in IseqProfile + /// Run a given callback with a mutable reference to every object in IseqProfile. pub fn each_object_mut(&mut self, callback: impl Fn(&mut VALUE)) { - for operands in &mut self.opnd_types { - for distribution in operands { + for entry in &mut self.entries { + for distribution in &mut entry.opnd_types { for ref mut profiled_type in distribution.each_item_mut() { // If the type is a GC object, call the callback callback(&mut profiled_type.class); } } } + + // Update CME references if they move during compaction. + for super_cme_values in self.super_cme.values_mut() { + for ref mut profiled_type in super_cme_values.each_item_mut() { + callback(&mut profiled_type.class) + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::cruby::*; + + #[test] + fn can_profile_block_handler() { + with_rubyvm(|| eval(" + def foo = yield + foo rescue 0 + foo rescue 0 + ")); } } diff --git a/zjit/src/state.rs b/zjit/src/state.rs index 194b02fc8d..da09d09314 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -1,19 +1,28 @@ -use crate::codegen::{gen_exit_trampoline, gen_function_stub_hit_trampoline}; -use crate::cruby::{self, rb_bug_panic_hook, rb_vm_insns_count, EcPtr, Qnil, VALUE}; +//! Runtime state of ZJIT. + +use crate::codegen::{gen_entry_trampoline, gen_exit_trampoline, gen_function_stub_hit_trampoline, gen_materialize_exit_trampoline, gen_materialize_exit_trampoline_with_counter}; +use crate::cruby::{self, rb_bug_panic_hook, rb_vm_insn_count, src_loc, EcPtr, Qnil, Qtrue, rb_profile_frames, rb_profile_frame_full_label, rb_profile_frame_absolute_path, rb_profile_frame_path, VALUE, VM_INSTRUCTION_SIZE, with_vm_lock, rust_str_to_id, rb_funcallv, rb_const_get, rb_cRubyVM}; use crate::cruby_methods; +use cruby::{ID, rb_callable_method_entry, get_def_method_serial, rb_gc_register_mark_object, ruby_str_to_rust_string_result}; +use std::sync::atomic::Ordering; use crate::invariants::Invariants; use crate::asm::CodeBlock; -use crate::options::get_option; -use crate::stats::Counters; +use crate::options::{get_option, rb_zjit_prepare_options}; +use crate::jit_frame::JITFrame; +use crate::stats::{Counters, InsnCounters, PerfettoTracer}; use crate::virtualmem::CodePtr; +use std::sync::atomic::AtomicUsize; +use std::collections::HashMap; +use std::ptr::null; +/// Shared trampoline to enter ZJIT. Not null when ZJIT is enabled. #[allow(non_upper_case_globals)] #[unsafe(no_mangle)] -pub static mut rb_zjit_enabled_p: bool = false; +pub static mut rb_zjit_entry: *const u8 = null(); /// Like rb_zjit_enabled_p, but for Rust code. pub fn zjit_enabled_p() -> bool { - unsafe { rb_zjit_enabled_p } + unsafe { rb_zjit_entry != null() } } /// Global state needed for code generation @@ -24,6 +33,12 @@ pub struct ZJITState { /// ZJIT statistics counters: Counters, + /// Side-exit counters + exit_counters: InsnCounters, + + /// Send fallback counters + send_fallback_counters: InsnCounters, + /// Assumptions that require invalidation invariants: Invariants, @@ -36,80 +51,139 @@ pub struct ZJITState { /// Trampoline to side-exit without restoring PC or the stack exit_trampoline: CodePtr, + /// Trampoline to materialize JIT frames before side-exiting + materialize_exit_trampoline: CodePtr, + + /// Trampoline to materialize JIT frames and increment exit_compilation_failure + materialize_exit_trampoline_with_counter: CodePtr, + /// Trampoline to call function_stub_hit function_stub_hit_trampoline: CodePtr, + + /// Counter pointers for full frame C functions + full_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for un-annotated C functions + not_annotated_frame_cfunc_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for all calls to any kind of C function from JIT code + ccall_counter_pointers: HashMap<String, Box<u64>>, + + /// Counter pointers for access counts of ISEQs accessed by JIT code + iseq_calls_count_pointers: HashMap<String, Box<u64>>, + + /// Perfetto tracer for --zjit-trace-exits + perfetto_tracer: Option<PerfettoTracer>, + + /// Frame metadata for ISEQ and C calls that are known at compile time + jit_frames: Vec<*mut JITFrame>, +} + +/// Tracks the initialization progress +enum InitializationState { + Uninitialized, + + /// At boot time, rb_zjit_init will be called regardless of whether + /// ZJIT is enabled, in this phase we initialize any states that must + /// be captured at during boot. + Initialized(cruby_methods::Annotations), + + /// When ZJIT is enabled, either during boot with `--zjit`, or lazily + /// at a later time with `RubyVM::ZJIT.enable`, we perform the rest + /// of the initialization steps and produce the `ZJITState` instance. + Enabled(ZJITState), + + /// Indicates that ZJITState::init has panicked. Should never be + /// encountered in practice since we abort immediately when that + /// happens. + Panicked, } /// Private singleton instance of the codegen globals -static mut ZJIT_STATE: Option<ZJITState> = None; +static mut ZJIT_STATE: InitializationState = InitializationState::Uninitialized; impl ZJITState { - /// Initialize the ZJIT globals - pub fn init() { - #[cfg(not(test))] - let mut cb = { - use crate::cruby::*; - use crate::options::*; + /// Initialize the ZJIT globals. Return the address of the JIT entry trampoline. + pub fn init() -> *const u8 { + use InitializationState::*; - let exec_mem_bytes: usize = get_option!(exec_mem_bytes); - let virt_block: *mut u8 = unsafe { rb_zjit_reserve_addr_space(64 * 1024 * 1024) }; - - // Memory protection syscalls need page-aligned addresses, so check it here. Assuming - // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the - // page size in bytes is a power of two 2¹⁹ or smaller. This is because the user - // requested size is half of mem_option × 2²⁰ as it's in MiB. - // - // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB - // (2¹⁶ bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. - let page_size = unsafe { rb_zjit_get_page_size() }; - assert_eq!( - virt_block as usize % page_size as usize, 0, - "Start of virtual address block should be page-aligned", - ); + let initialization_state = unsafe { + std::mem::replace(&mut ZJIT_STATE, Panicked) + }; + + let Initialized(method_annotations) = initialization_state else { + panic!("rb_zjit_init was never called"); + }; + let mut cb = { + use crate::options::*; use crate::virtualmem::*; - use std::ptr::NonNull; use std::rc::Rc; use std::cell::RefCell; - let mem_block = VirtualMem::new( - crate::virtualmem::sys::SystemAllocator {}, - page_size, - NonNull::new(virt_block).unwrap(), - exec_mem_bytes, - exec_mem_bytes, // TODO: change this to --zjit-mem-size (Shopify/ruby#686) - ); + let mem_block = VirtualMem::alloc(get_option!(exec_mem_bytes), Some(get_option!(mem_bytes))); let mem_block = Rc::new(RefCell::new(mem_block)); - CodeBlock::new(mem_block.clone(), get_option!(dump_disasm)) + CodeBlock::new(mem_block.clone(), get_option_ref!(dump_disasm).is_some()) }; - #[cfg(test)] - let mut cb = CodeBlock::new_dummy(); + let entry_trampoline = gen_entry_trampoline(&mut cb).unwrap().raw_ptr(&cb); let exit_trampoline = gen_exit_trampoline(&mut cb).unwrap(); + let materialize_exit_trampoline = gen_materialize_exit_trampoline(&mut cb, exit_trampoline).unwrap(); let function_stub_hit_trampoline = gen_function_stub_hit_trampoline(&mut cb).unwrap(); + let perfetto_tracer = if get_option!(trace_side_exits).is_some() || get_option!(trace_compiles) || get_option!(trace_invalidation) { + Some(PerfettoTracer::new()) + } else { + None + }; + // Initialize the codegen globals instance let zjit_state = ZJITState { code_block: cb, counters: Counters::default(), + exit_counters: [0; VM_INSTRUCTION_SIZE as usize], + send_fallback_counters: [0; VM_INSTRUCTION_SIZE as usize], invariants: Invariants::default(), assert_compiles: false, - method_annotations: cruby_methods::init(), + method_annotations, exit_trampoline, + materialize_exit_trampoline, + materialize_exit_trampoline_with_counter: materialize_exit_trampoline, function_stub_hit_trampoline, + full_frame_cfunc_counter_pointers: HashMap::new(), + not_annotated_frame_cfunc_counter_pointers: HashMap::new(), + ccall_counter_pointers: HashMap::new(), + iseq_calls_count_pointers: HashMap::new(), + perfetto_tracer, + jit_frames: vec![], }; - unsafe { ZJIT_STATE = Some(zjit_state); } + unsafe { ZJIT_STATE = Enabled(zjit_state); } + + // With --zjit-stats, use a different trampoline on function stub exits + // to count exit_compilation_failure. Note that the trampoline code depends + // on the counter, so ZJIT_STATE needs to be initialized first. + if get_option!(stats) { + let cb = ZJITState::get_code_block(); + let code_ptr = gen_materialize_exit_trampoline_with_counter(cb, materialize_exit_trampoline).unwrap(); + ZJITState::get_instance().materialize_exit_trampoline_with_counter = code_ptr; + } + + entry_trampoline } /// Return true if zjit_state has been initialized pub fn has_instance() -> bool { - unsafe { ZJIT_STATE.as_mut().is_some() } + matches!(unsafe { &ZJIT_STATE }, InitializationState::Enabled(_)) } /// Get a mutable reference to the codegen globals instance fn get_instance() -> &'static mut ZJITState { - unsafe { ZJIT_STATE.as_mut().unwrap() } + if let InitializationState::Enabled(instance) = unsafe { &mut ZJIT_STATE } { + instance + } else { + panic!("ZJITState::get_instance called when ZJIT is not enabled") + } } /// Get a mutable reference to the inline code block @@ -122,6 +196,10 @@ impl ZJITState { &mut ZJITState::get_instance().invariants } + pub fn get_jit_frames() -> &'static mut Vec<*mut JITFrame> { + &mut ZJITState::get_instance().jit_frames + } + pub fn get_method_annotations() -> &'static cruby_methods::Annotations { &ZJITState::get_instance().method_annotations } @@ -137,11 +215,47 @@ impl ZJITState { instance.assert_compiles = true; } + /// Stop asserting successful compilation + pub fn disable_assert_compiles() { + let instance = ZJITState::get_instance(); + instance.assert_compiles = false; + } + /// Get a mutable reference to counters for ZJIT stats pub fn get_counters() -> &'static mut Counters { &mut ZJITState::get_instance().counters } + /// Get a mutable reference to side-exit counters + pub fn get_exit_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().exit_counters + } + + /// Get a mutable reference to fallback counters + pub fn get_send_fallback_counters() -> &'static mut InsnCounters { + &mut ZJITState::get_instance().send_fallback_counters + } + + /// Get a mutable reference to full frame cfunc counter pointers + pub fn get_not_inlined_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().full_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to non-annotated cfunc counter pointers + pub fn get_not_annotated_cfunc_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().not_annotated_frame_cfunc_counter_pointers + } + + /// Get a mutable reference to ccall counter pointers + pub fn get_ccall_counter_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().ccall_counter_pointers + } + + /// Get a mutable reference to iseq access count pointers + pub fn get_iseq_calls_count_pointers() -> &'static mut HashMap<String, Box<u64>> { + &mut ZJITState::get_instance().iseq_calls_count_pointers + } + /// Was --zjit-save-compiled-iseqs specified? pub fn should_log_compiled_iseqs() -> bool { get_option!(log_compiled_iseqs).is_some() @@ -155,12 +269,12 @@ impl ZJITState { let mut file = match std::fs::OpenOptions::new().create(true).append(true).open(filename) { Ok(f) => f, Err(e) => { - eprintln!("ZJIT: Failed to create file '{}': {}", filename, e); + eprintln!("ZJIT: Failed to create file '{}': {}", filename.display(), e); return; } }; - if let Err(e) = writeln!(file, "{}", iseq_name) { - eprintln!("ZJIT: Failed to write to file '{}': {}", filename, e); + if let Err(e) = writeln!(file, "{iseq_name}") { + eprintln!("ZJIT: Failed to write to file '{}': {}", filename.display(), e); } } @@ -179,42 +293,249 @@ impl ZJITState { ZJITState::get_instance().exit_trampoline } + /// Return a code pointer to the materialize_exit trampoline + pub fn get_materialize_exit_trampoline() -> CodePtr { + ZJITState::get_instance().materialize_exit_trampoline + } + + /// Return a code pointer to the materialize_exit trampoline for function stubs + pub fn get_materialize_exit_trampoline_with_counter() -> CodePtr { + ZJITState::get_instance().materialize_exit_trampoline_with_counter + } + /// Return a code pointer to the function stub hit trampoline pub fn get_function_stub_hit_trampoline() -> CodePtr { ZJITState::get_instance().function_stub_hit_trampoline } + + /// Get a mutable reference to the Perfetto tracer + pub fn get_tracer() -> Option<&'static mut PerfettoTracer> { + if !ZJITState::has_instance() { return None; } + ZJITState::get_instance().perfetto_tracer.as_mut() + } } -/// Initialize ZJIT +/// The `::RubyVM::ZJIT` module. +pub static ZJIT_MODULE: AtomicUsize = AtomicUsize::new(!0); +/// Serial of the canonical version of `induce_side_exit!` right after VM boot. +pub static INDUCE_SIDE_EXIT_SERIAL: AtomicUsize = AtomicUsize::new(!0); +/// Serial of the canonical version of `induce_compile_failure!` right after VM boot. +pub static INDUCE_COMPILE_FAILURE_SERIAL: AtomicUsize = AtomicUsize::new(!0); +/// Serial of the canonical version of `induce_breakpoint!` right after VM boot. +pub static INDUCE_BREAKPOINT_SERIAL: AtomicUsize = AtomicUsize::new(!0); + +/// Check if a method, `method_id`, currently exists on `ZJIT.singleton_class` and has the `expected_serial`. +pub fn zjit_module_method_match_serial(method_id: ID, expected_serial: &AtomicUsize) -> bool { + let zjit_module_singleton = VALUE(ZJIT_MODULE.load(Ordering::Relaxed)).class_of(); + let cme = unsafe { rb_callable_method_entry(zjit_module_singleton, method_id) }; + if cme.is_null() { + false + } else { + let serial = unsafe { get_def_method_serial((*cme).def) }; + serial == expected_serial.load(std::sync::atomic::Ordering::Relaxed) + } +} + +/// Initialize IDs and annotate builtin C method entries. +/// Must be called at boot before ruby_init_prelude() since the prelude +/// could redefine core methods (e.g. Kernel.prepend via bundler). #[unsafe(no_mangle)] -pub extern "C" fn rb_zjit_init() { +pub extern "C" fn rb_zjit_init_builtin_cmes() { + use InitializationState::*; + + debug_assert!( + matches!(unsafe { &ZJIT_STATE }, Uninitialized), + "rb_zjit_init_builtin_cmes should only be called once during boot", + ); + + cruby::ids::init(); + + let method_annotations = cruby_methods::init(); + + unsafe { ZJIT_STATE = Initialized(method_annotations); } + + // Boot time setup for compiler directives + unsafe { + let zjit_module = rb_const_get(rb_cRubyVM, rust_str_to_id("ZJIT")); + + let cme = rb_callable_method_entry(zjit_module.class_of(), ID!(induce_side_exit_bang)); + assert!(! cme.is_null(), "RubyVM::ZJIT.induce_side_exit! should exist on boot"); + let serial = get_def_method_serial((*cme).def) ; + INDUCE_SIDE_EXIT_SERIAL.store(serial, Ordering::Relaxed); + + let cme = rb_callable_method_entry(zjit_module.class_of(), ID!(induce_compile_failure_bang)); + assert!(! cme.is_null(), "RubyVM::ZJIT.induce_compile_failure! should exist on boot"); + let serial = get_def_method_serial((*cme).def) ; + INDUCE_COMPILE_FAILURE_SERIAL.store(serial, Ordering::Relaxed); + + let cme = rb_callable_method_entry(zjit_module.class_of(), ID!(induce_breakpoint_bang)); + assert!(! cme.is_null(), "RubyVM::ZJIT.induce_breakpoint! should exist on boot"); + let serial = get_def_method_serial((*cme).def) ; + INDUCE_BREAKPOINT_SERIAL.store(serial, Ordering::Relaxed); + + // Root and pin the module since we'll be doing object identity comparisons. + ZJIT_MODULE.store(zjit_module.0, Ordering::Relaxed); + rb_gc_register_mark_object(zjit_module); + } +} + +/// Initialize ZJIT at boot. This is called even if ZJIT is disabled. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_init(zjit_enabled: bool) { + // If --zjit, enable ZJIT immediately + if zjit_enabled { + zjit_enable(); + } +} + +/// Enable ZJIT compilation. +fn zjit_enable() { + // Call ZJIT hooks before enabling ZJIT to avoid compiling the hooks themselves + unsafe { + let zjit = rb_const_get(rb_cRubyVM, rust_str_to_id("ZJIT")); + rb_funcallv(zjit, rust_str_to_id("call_jit_hooks"), 0, std::ptr::null()); + } + // Catch panics to avoid UB for unwinding into C frames. // See https://doc.rust-lang.org/nomicon/exception-safety.html let result = std::panic::catch_unwind(|| { // Initialize ZJIT states - cruby::ids::init(); - ZJITState::init(); + let zjit_entry = ZJITState::init(); // Install a panic hook for ZJIT rb_bug_panic_hook(); // Discard the instruction count for boot which we never compile - unsafe { rb_vm_insns_count = 0; } + unsafe { rb_vm_insn_count = 0; } // ZJIT enabled and initialized successfully - assert!(unsafe{ !rb_zjit_enabled_p }); - unsafe { rb_zjit_enabled_p = true; } + assert!(unsafe{ rb_zjit_entry == null() }); + unsafe { rb_zjit_entry = zjit_entry; } }); if result.is_err() { - println!("ZJIT: zjit_init() panicked. Aborting."); + println!("ZJIT: zjit_enable() panicked. Aborting."); std::process::abort(); } } +/// Enable ZJIT compilation, returning Qtrue if ZJIT was previously disabled +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_enable(_ec: EcPtr, _self: VALUE) -> VALUE { + with_vm_lock(src_loc!(), || { + // Options would not have been initialized during boot if no flags were specified + rb_zjit_prepare_options(); + + // Initialize and enable ZJIT + zjit_enable(); + + // Add "+ZJIT" to RUBY_DESCRIPTION + unsafe { + unsafe extern "C" { + fn ruby_set_zjit_description(); + } + ruby_set_zjit_description(); + } + + Qtrue + }) +} + /// Assert that any future ZJIT compilation will return a function pointer (not fail to compile) #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_assert_compiles(_ec: EcPtr, _self: VALUE) -> VALUE { ZJITState::enable_assert_compiles(); Qnil } + +/// Resolve a profile frame VALUE to a human-readable "label (path)" string. +fn resolve_frame_label(frame: VALUE) -> String { + unsafe { + let label_str = ruby_str_to_rust_string_result(rb_profile_frame_full_label(frame)).unwrap_or("<unknown>".into()); + + let path = rb_profile_frame_absolute_path(frame); + let path = if path.nil_p() { rb_profile_frame_path(frame) } else { path }; + let path_str = ruby_str_to_rust_string_result(path).unwrap_or("<unknown>".into()); + + format!("{label_str} ({path_str})") + } +} + +/// Record a backtrace with ZJIT side exits as a Perfetto trace event +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_record_exit_stack(reason: *const std::ffi::c_char) { + if !zjit_enabled_p() || get_option!(trace_side_exits).is_none() { + return; + } + + let tracer = match ZJITState::get_tracer() { + Some(t) => t, + None => return, + }; + + // When `trace_side_exits_sample_interval` is non-zero, apply sampling. + if get_option!(trace_side_exits_sample_interval) != 0 { + if tracer.skipped_samples < get_option!(trace_side_exits_sample_interval) { + tracer.skipped_samples += 1; + return; + } else { + tracer.skipped_samples = 0; + } + } + + // Collect profile frames + let frames = capture_ruby_frames(); + + // Get the reason string + let reason_str = if reason.is_null() { + "unknown" + } else { + unsafe { std::ffi::CStr::from_ptr(reason).to_str().unwrap_or("unknown") } + }; + + tracer.write_event("side_exit", reason_str, &frames); +} + +/// Wrap a closure in a Perfetto duration event with category "invalidation" +/// and a Ruby backtrace captured on the begin event. +pub fn trace_invalidation<F, R>(reason: &str, func: F) -> R where F: FnOnce() -> R { + if !get_option!(trace_invalidation) { + return func(); + } + + // Capture backtrace and emit begin event before patching + let frames = capture_ruby_frames(); + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_begin("invalidation", reason, ts, &frames); + } + + let result = func(); + + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_end("invalidation", reason, ts); + } + result +} + +/// Capture the current Ruby call stack as human-readable frame labels. +fn capture_ruby_frames() -> Vec<String> { + const BUFF_LEN: usize = 2048; + let mut frames_buffer = vec![VALUE(0_usize); BUFF_LEN]; + let mut lines_buffer = vec![0i32; BUFF_LEN]; + + let stack_length = unsafe { + rb_profile_frames( + 0, + BUFF_LEN as i32, + frames_buffer.as_mut_ptr(), + lines_buffer.as_mut_ptr(), + ) + }; + + // Resolve each frame to a human-readable string (top frame first) + (0..stack_length as usize) + .map(|i| resolve_frame_label(frames_buffer[i])) + .collect() +} diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index ce185597c4..57320a02e7 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -1,18 +1,51 @@ +//! Counters and associated methods for events when ZJIT is run. + use std::time::Instant; +use std::sync::atomic::Ordering; +use crate::options::OPTIONS; + +// test binaries always bring it in as a cargo dependency +#[cfg(all(feature = "stats_allocator", not(test)))] +#[path = "../../jit/src/lib.rs"] +mod jit; -use crate::{cruby::*, options::get_option, state::{zjit_enabled_p, ZJITState}}; +use crate::{cruby::*, hir::ParseError, options::get_option, state::{zjit_enabled_p, ZJITState}}; macro_rules! make_counters { ( default { $($default_counter_name:ident,)+ } + exit { + $($exit_counter_name:ident,)+ + } + dynamic_send { + $($dynamic_send_counter_name:ident,)+ + } + optimized_send { + $($optimized_send_counter_name:ident,)+ + } + dynamic_setivar { + $($dynamic_setivar_counter_name:ident,)+ + } + dynamic_getivar { + $($dynamic_getivar_counter_name:ident,)+ + } + dynamic_definedivar { + $($dynamic_definedivar_counter_name:ident,)+ + } $($counter_name:ident,)+ ) => { /// Struct containing the counter values #[derive(Default, Debug)] pub struct Counters { $(pub $default_counter_name: u64,)+ + $(pub $exit_counter_name: u64,)+ + $(pub $dynamic_send_counter_name: u64,)+ + $(pub $optimized_send_counter_name: u64,)+ + $(pub $dynamic_setivar_counter_name: u64,)+ + $(pub $dynamic_getivar_counter_name: u64,)+ + $(pub $dynamic_definedivar_counter_name: u64,)+ $(pub $counter_name: u64,)+ } @@ -21,14 +54,40 @@ macro_rules! make_counters { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Counter { $($default_counter_name,)+ + $($exit_counter_name,)+ + $($dynamic_send_counter_name,)+ + $($optimized_send_counter_name,)+ + $($dynamic_setivar_counter_name,)+ + $($dynamic_getivar_counter_name,)+ + $($dynamic_definedivar_counter_name,)+ $($counter_name,)+ } impl Counter { - pub fn name(&self) -> String { + pub fn name(&self) -> &'static str { match self { - $( Counter::$default_counter_name => stringify!($default_counter_name).to_string(), )+ - $( Counter::$counter_name => stringify!($counter_name).to_string(), )+ + $( Counter::$default_counter_name => stringify!($default_counter_name), )+ + $( Counter::$exit_counter_name => stringify!($exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => stringify!($dynamic_send_counter_name), )+ + $( Counter::$optimized_send_counter_name => stringify!($optimized_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => stringify!($dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => stringify!($dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => stringify!($dynamic_definedivar_counter_name), )+ + $( Counter::$counter_name => stringify!($counter_name), )+ + } + } + + pub fn get(name: &str) -> Option<Counter> { + match name { + $( stringify!($default_counter_name) => Some(Counter::$default_counter_name), )+ + $( stringify!($exit_counter_name) => Some(Counter::$exit_counter_name), )+ + $( stringify!($dynamic_send_counter_name) => Some(Counter::$dynamic_send_counter_name), )+ + $( stringify!($optimized_send_counter_name) => Some(Counter::$optimized_send_counter_name), )+ + $( stringify!($dynamic_setivar_counter_name) => Some(Counter::$dynamic_setivar_counter_name), )+ + $( stringify!($dynamic_getivar_counter_name) => Some(Counter::$dynamic_getivar_counter_name), )+ + $( stringify!($dynamic_definedivar_counter_name) => Some(Counter::$dynamic_definedivar_counter_name), )+ + $( stringify!($counter_name) => Some(Counter::$counter_name), )+ + _ => None, } } } @@ -38,15 +97,56 @@ macro_rules! make_counters { let counters = $crate::state::ZJITState::get_counters(); match counter { $( Counter::$default_counter_name => std::ptr::addr_of_mut!(counters.$default_counter_name), )+ + $( Counter::$exit_counter_name => std::ptr::addr_of_mut!(counters.$exit_counter_name), )+ + $( Counter::$dynamic_send_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_send_counter_name), )+ + $( Counter::$dynamic_setivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_setivar_counter_name), )+ + $( Counter::$dynamic_getivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_getivar_counter_name), )+ + $( Counter::$dynamic_definedivar_counter_name => std::ptr::addr_of_mut!(counters.$dynamic_definedivar_counter_name), )+ + $( Counter::$optimized_send_counter_name => std::ptr::addr_of_mut!(counters.$optimized_send_counter_name), )+ $( Counter::$counter_name => std::ptr::addr_of_mut!(counters.$counter_name), )+ } } - /// The list of counters that are available without --zjit-stats. + /// List of counters that are available without --zjit-stats. /// They are incremented only by `incr_counter()` and don't use `gen_incr_counter()`. pub const DEFAULT_COUNTERS: &'static [Counter] = &[ $( Counter::$default_counter_name, )+ ]; + + /// List of other counters that are summed as side_exit_count. + pub const EXIT_COUNTERS: &'static [Counter] = &[ + $( Counter::$exit_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_send_count. + pub const DYNAMIC_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_send_counter_name, )+ + ]; + + /// List of other counters that are summed as optimized_send_count. + pub const OPTIMIZED_SEND_COUNTERS: &'static [Counter] = &[ + $( Counter::$optimized_send_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_setivar_count. + pub const DYNAMIC_SETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_setivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_getivar_count. + pub const DYNAMIC_GETIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_getivar_counter_name, )+ + ]; + + /// List of other counters that are summed as dynamic_definedivar_count. + pub const DYNAMIC_DEFINEDIVAR_COUNTERS: &'static [Counter] = &[ + $( Counter::$dynamic_definedivar_counter_name, )+ + ]; + + /// List of other counters that are available only for --zjit-stats. + pub const OTHER_COUNTERS: &'static [Counter] = &[ + $( Counter::$counter_name, )+ + ]; } } @@ -54,22 +154,660 @@ macro_rules! make_counters { make_counters! { // Default counters that are available without --zjit-stats default { + compiled_iseq_count, + failed_iseq_count, + skipped_native_stack_full, + compile_time_ns, profile_time_ns, gc_time_ns, invalidation_time_ns, + + compiled_side_exit_count, + side_exit_size, + compile_side_exit_time_ns, + + compile_hir_time_ns, + compile_hir_build_time_ns, + compile_hir_strength_reduce_time_ns, + compile_hir_optimize_load_store_time_ns, + compile_hir_canonicalize_time_ns, + compile_hir_fold_constants_time_ns, + compile_hir_clean_cfg_time_ns, + compile_hir_remove_redundant_patch_points_time_ns, + compile_hir_remove_duplicate_check_interrupts_time_ns, + compile_hir_eliminate_dead_code_time_ns, + compile_lir_time_ns, } + // Exit counters that are summed as side_exit_count + exit { + // exit_: Side exits reasons + exit_compile_error, + exit_unhandled_newarray_send_min, + exit_unhandled_newarray_send_hash, + exit_unhandled_newarray_send_pack, + exit_unhandled_newarray_send_pack_buffer, + exit_unhandled_newarray_send_unknown, + exit_unhandled_duparray_send, + exit_unhandled_tailcall, + exit_unhandled_splat, + exit_unhandled_kwarg, + exit_unhandled_block_arg, + exit_unknown_special_variable, + exit_unhandled_hir_insn, + exit_unhandled_yarv_insn, + exit_fixnum_add_overflow, + exit_fixnum_sub_overflow, + exit_fixnum_mult_overflow, + exit_fixnum_lshift_overflow, + exit_fixnum_mod_by_zero, + exit_fixnum_div_by_zero, + exit_box_fixnum_overflow, + exit_guard_type_failure, + exit_guard_bit_equals_failure, + exit_guard_int_equals_failure, + exit_guard_shape_failure, + exit_expandarray_failure, + exit_guard_not_frozen_failure, + exit_guard_not_shared_failure, + exit_guard_less_failure, + exit_guard_greater_eq_failure, + exit_guard_super_method_entry, + exit_patchpoint_bop_redefined, + exit_patchpoint_method_redefined, + exit_patchpoint_stable_constant_names, + exit_patchpoint_no_tracepoint, + exit_patchpoint_no_ep_escape, + exit_patchpoint_single_ractor_mode, + exit_patchpoint_no_singleton_class, + exit_patchpoint_root_box_only, + exit_callee_side_exit, + exit_obj_to_string_fallback, + exit_interrupt, + exit_stackoverflow, + exit_block_param_proxy_not_iseq_or_ifunc, + exit_block_param_proxy_not_nil, + exit_block_param_proxy_fallback_miss, + exit_block_param_proxy_profile_not_covered, + exit_block_param_wb_required, + exit_too_many_keyword_parameters, + exit_no_profile_send, + exit_splatkw_not_nil_or_hash, + exit_splatkw_polymorphic, + exit_splatkw_not_profiled, + exit_directive_induced, + exit_send_while_tracing, + exit_invokeblock_not_ifunc, + } + + // Send fallback counters that are summed as dynamic_send_count + dynamic_send { + // send_fallback_: Fallback reasons for send-ish instructions + send_fallback_send_without_block_polymorphic, + send_fallback_send_without_block_megamorphic, + send_fallback_send_without_block_no_profiles, + send_fallback_send_without_block_cfunc_not_variadic, + send_fallback_send_without_block_cfunc_array_variadic, + send_fallback_send_without_block_not_optimized_method_type, + send_fallback_send_without_block_not_optimized_method_type_optimized, + send_fallback_send_without_block_not_optimized_need_permission, + send_fallback_too_many_args_for_lir, + send_fallback_send_without_block_bop_redefined, + send_fallback_send_without_block_operands_not_fixnum, + send_fallback_send_without_block_polymorphic_fallback, + send_fallback_send_without_block_direct_keyword_mismatch, + send_fallback_send_without_block_direct_keyword_count_mismatch, + send_fallback_send_without_block_direct_missing_keyword, + send_fallback_send_without_block_direct_too_many_keywords, + send_fallback_send_polymorphic, + send_fallback_send_megamorphic, + send_fallback_send_no_profiles, + send_fallback_send_not_optimized_method_type, + send_fallback_send_not_optimized_need_permission, + send_fallback_ccall_with_frame_too_many_args, + send_fallback_argc_param_mismatch, + // The call has at least one feature on the caller or callee side + // that the optimizer does not support. + send_fallback_one_or_more_complex_arg_pass, + // Caller has keyword arguments but callee doesn't expect them. + send_fallback_unexpected_keyword_args, + // Singleton class previously created for receiver class. + send_fallback_singleton_class_seen, + send_fallback_bmethod_non_iseq_proc, + send_fallback_obj_to_string_not_string, + send_fallback_send_cfunc_variadic, + send_fallback_send_cfunc_array_variadic, + send_fallback_super_call_with_block, + send_fallback_super_from_block, + send_fallback_super_class_not_found, + send_fallback_super_complex_args_pass, + send_fallback_super_fallback_no_profile, + send_fallback_super_not_optimized_method_type, + send_fallback_super_polymorphic, + send_fallback_super_target_not_found, + send_fallback_super_target_complex_args_pass, + send_fallback_cannot_send_direct, + send_fallback_invokeblock_not_specialized, + send_fallback_sendforward_not_specialized, + send_fallback_invokesuperforward_not_specialized, + send_fallback_single_ractor_mode_required, + send_fallback_uncategorized, + } + + // Optimized send counters that are summed as optimized_send_count + optimized_send { + iseq_optimized_send_count, + inline_cfunc_optimized_send_count, + inline_iseq_optimized_send_count, + non_variadic_cfunc_optimized_send_count, + variadic_cfunc_optimized_send_count, + } + + // Ivar fallback counters that are summed as dynamic_setivar_count + dynamic_setivar { + // setivar_fallback_: Fallback reasons for dynamic setivar instructions + setivar_fallback_not_monomorphic, + setivar_fallback_immediate, + setivar_fallback_not_t_object, + setivar_fallback_complex, + setivar_fallback_frozen, + setivar_fallback_shape_transition, + setivar_fallback_new_shape_complex, + setivar_fallback_new_shape_needs_extension, + } + + // Ivar fallback counters that are summed as dynamic_getivar_count + dynamic_getivar { + // getivar_fallback_: Fallback reasons for dynamic getivar instructions + getivar_fallback_not_monomorphic, + getivar_fallback_immediate, + getivar_fallback_not_t_object, + getivar_fallback_complex, + } + + // Ivar fallback counters that are summed as dynamic_definedivar_count + dynamic_definedivar { + // definedivar_fallback_: Fallback reasons for dynamic definedivar instructions + definedivar_fallback_not_monomorphic, + definedivar_fallback_immediate, + definedivar_fallback_not_t_object, + definedivar_fallback_complex, + } + + // compile_error_: Compile error reasons + compile_error_iseq_version_limit_reached, + compile_error_iseq_stack_too_large, + compile_error_native_stack_too_large, + compile_error_exception_handler, + compile_error_out_of_memory, + compile_error_label_linking_failure, + compile_error_jit_to_jit_optional, + compile_error_register_spill_on_ccall, + compile_error_register_spill_on_alloc, + compile_error_parse_stack_underflow, + compile_error_parse_malformed_iseq, + compile_error_parse_not_allowed, + compile_error_parse_directive_induced, + compile_error_validation_block_has_no_terminator, + compile_error_validation_terminator_not_at_end, + compile_error_validation_mismatched_block_arity, + compile_error_validation_jump_target_not_in_rpo, + compile_error_validation_operand_not_defined, + compile_error_validation_duplicate_instruction, + compile_error_validation_type_check_failure, + compile_error_validation_misc_validation_error, + + // unhandled_hir_insn_: Unhandled HIR instructions + unhandled_hir_insn_array_max, + unhandled_hir_insn_fixnum_div, + unhandled_hir_insn_throw, + unhandled_hir_insn_invokebuiltin, + unhandled_hir_insn_unknown, + // The number of times YARV instructions are executed on JIT code - zjit_insns_count, + zjit_insn_count, + + // Method call def_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_iseq, + unspecialized_send_without_block_def_type_cfunc, + unspecialized_send_without_block_def_type_attrset, + unspecialized_send_without_block_def_type_ivar, + unspecialized_send_without_block_def_type_bmethod, + unspecialized_send_without_block_def_type_zsuper, + unspecialized_send_without_block_def_type_alias, + unspecialized_send_without_block_def_type_undef, + unspecialized_send_without_block_def_type_not_implemented, + unspecialized_send_without_block_def_type_optimized, + unspecialized_send_without_block_def_type_missing, + unspecialized_send_without_block_def_type_refined, + unspecialized_send_without_block_def_type_null, + + // Method call optimized_type related to send without block fallback to dynamic dispatch + unspecialized_send_without_block_def_type_optimized_send, + unspecialized_send_without_block_def_type_optimized_call, + unspecialized_send_without_block_def_type_optimized_block_call, + unspecialized_send_without_block_def_type_optimized_struct_aref, + unspecialized_send_without_block_def_type_optimized_struct_aset, + + // Method call def_type related to send fallback to dynamic dispatch + unspecialized_send_def_type_iseq, + unspecialized_send_def_type_cfunc, + unspecialized_send_def_type_attrset, + unspecialized_send_def_type_ivar, + unspecialized_send_def_type_bmethod, + unspecialized_send_def_type_zsuper, + unspecialized_send_def_type_alias, + unspecialized_send_def_type_undef, + unspecialized_send_def_type_not_implemented, + unspecialized_send_def_type_optimized, + unspecialized_send_def_type_missing, + unspecialized_send_def_type_refined, + unspecialized_send_def_type_null, + + // Super call def_type related to send fallback to dynamic dispatch + unspecialized_super_def_type_iseq, + unspecialized_super_def_type_cfunc, + unspecialized_super_def_type_attrset, + unspecialized_super_def_type_ivar, + unspecialized_super_def_type_bmethod, + unspecialized_super_def_type_zsuper, + unspecialized_super_def_type_alias, + unspecialized_super_def_type_undef, + unspecialized_super_def_type_not_implemented, + unspecialized_super_def_type_optimized, + unspecialized_super_def_type_missing, + unspecialized_super_def_type_refined, + unspecialized_super_def_type_null, + + // Unsupported parameter features + complex_arg_pass_param_rest, + complex_arg_pass_param_post, + complex_arg_pass_param_kwrest, + complex_arg_pass_param_block, + complex_arg_pass_param_forwardable, + complex_arg_pass_accepts_no_block, + complex_arg_pass_does_not_use_block, + + // Unsupported caller side features + complex_arg_pass_caller_splat, + complex_arg_pass_caller_blockarg, + complex_arg_pass_caller_kwarg, + complex_arg_pass_caller_kw_splat, + complex_arg_pass_caller_tailcall, + complex_arg_pass_caller_super, + complex_arg_pass_caller_zsuper, + complex_arg_pass_caller_forwarding, + + // Writes to the VM frame + vm_write_jit_frame_count, + vm_write_sp_count, + vm_write_locals_count, + vm_write_stack_count, + vm_write_to_parent_iseq_local_count, + // TODO(max): Implement + // vm_reify_stack_count, + + // The number of times we ran a dynamic check + guard_type_count, + guard_shape_count, + + load_field_count, + store_field_count, + + invokeblock_handler_monomorphic_iseq, + invokeblock_handler_monomorphic_ifunc, + invokeblock_handler_monomorphic_other, + invokeblock_handler_polymorphic, + invokeblock_handler_megamorphic, + invokeblock_handler_no_profiles, + + getblockparamproxy_handler_iseq, + getblockparamproxy_handler_ifunc, + getblockparamproxy_handler_symbol, + getblockparamproxy_handler_proc, + getblockparamproxy_handler_nil, + getblockparamproxy_handler_polymorphic, + getblockparamproxy_handler_megamorphic, + getblockparamproxy_handler_no_profiles, } /// Increase a counter by a specified amount -fn incr_counter(counter: Counter, amount: u64) { +pub fn incr_counter_by(counter: Counter, amount: u64) { let ptr = counter_ptr(counter); unsafe { *ptr += amount; } } +/// Decrease a counter by a specified amount +pub fn decr_counter_by(counter: Counter, amount: u64) { + let ptr = counter_ptr(counter); + unsafe { *ptr -= amount; } +} + +/// Increment a counter by its identifier +macro_rules! incr_counter { + ($counter_name:ident) => { + $crate::stats::incr_counter_by($crate::stats::Counter::$counter_name, 1) + } +} +pub(crate) use incr_counter; + +/// The number of side exits from each YARV instruction +pub type InsnCounters = [u64; VM_INSTRUCTION_SIZE as usize]; + +/// Return a raw pointer to the exit counter for a given YARV opcode +pub fn exit_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let exit_counters = ZJITState::get_exit_counters(); + unsafe { exit_counters.get_unchecked_mut(opcode as usize) } +} + +/// Return a raw pointer to the fallback counter for a given YARV opcode +pub fn send_fallback_counter_ptr_for_opcode(opcode: u32) -> *mut u64 { + let fallback_counters = ZJITState::get_send_fallback_counters(); + unsafe { fallback_counters.get_unchecked_mut(opcode as usize) } +} + +/// Reason why ZJIT failed to produce any JIT code +#[derive(Clone, Debug, PartialEq)] +pub enum CompileError { + IseqVersionLimitReached, + IseqStackTooLarge, + NativeStackTooLarge, + ExceptionHandler, + OutOfMemory, + ParseError(ParseError), + /// When a ZJIT function is too large, the branches may have + /// offsets that don't fit in one instruction. We error in + /// error that case. + LabelLinkingFailure, +} + +/// Return a raw pointer to the exit counter for a given CompileError +pub fn exit_counter_for_compile_error(compile_error: &CompileError) -> Counter { + use crate::hir::ParseError::*; + use crate::hir::ValidationError::*; + use crate::stats::CompileError::*; + use crate::stats::Counter::*; + match compile_error { + IseqVersionLimitReached => compile_error_iseq_version_limit_reached, + IseqStackTooLarge => compile_error_iseq_stack_too_large, + NativeStackTooLarge => compile_error_native_stack_too_large, + ExceptionHandler => compile_error_exception_handler, + OutOfMemory => compile_error_out_of_memory, + LabelLinkingFailure => compile_error_label_linking_failure, + ParseError(parse_error) => match parse_error { + StackUnderflow(_) => compile_error_parse_stack_underflow, + MalformedIseq(_) => compile_error_parse_malformed_iseq, + NotAllowed => compile_error_parse_not_allowed, + DirectiveInduced => compile_error_parse_directive_induced, + Validation(validation) => match validation { + BlockHasNoTerminator(_) => compile_error_validation_block_has_no_terminator, + TerminatorNotAtEnd(_, _, _) => compile_error_validation_terminator_not_at_end, + MismatchedBlockArity(_, _, _) => compile_error_validation_mismatched_block_arity, + JumpTargetNotInRPO(_) => compile_error_validation_jump_target_not_in_rpo, + OperandNotDefined(_, _, _) => compile_error_validation_operand_not_defined, + DuplicateInstruction(_, _) => compile_error_validation_duplicate_instruction, + MismatchedOperandType(..) => compile_error_validation_type_check_failure, + MiscValidationError(..) => compile_error_validation_misc_validation_error, + }, + } + } +} + +pub fn exit_counter_for_unhandled_hir_insn(insn: &crate::hir::Insn) -> Counter { + use crate::hir::Insn::*; + use crate::stats::Counter::*; + match insn { + ArrayMax { .. } => unhandled_hir_insn_array_max, + FixnumDiv { .. } => unhandled_hir_insn_fixnum_div, + Throw { .. } => unhandled_hir_insn_throw, + InvokeBuiltin { .. } => unhandled_hir_insn_invokebuiltin, + _ => unhandled_hir_insn_unknown, + } +} + +pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { + use crate::hir::SideExitReason::*; + use crate::hir::CallType::*; + use crate::hir::Invariant; + use crate::stats::Counter::*; + match reason { + UnhandledNewarraySend(send_type) => match send_type { + VM_OPT_NEWARRAY_SEND_MIN => exit_unhandled_newarray_send_min, + VM_OPT_NEWARRAY_SEND_HASH => exit_unhandled_newarray_send_hash, + VM_OPT_NEWARRAY_SEND_PACK => exit_unhandled_newarray_send_pack, + VM_OPT_NEWARRAY_SEND_PACK_BUFFER => exit_unhandled_newarray_send_pack_buffer, + _ => exit_unhandled_newarray_send_unknown, + } + UnhandledDuparraySend(_) => exit_unhandled_duparray_send, + UnhandledCallType(Tailcall) => exit_unhandled_tailcall, + UnhandledCallType(Splat) => exit_unhandled_splat, + UnhandledCallType(Kwarg) => exit_unhandled_kwarg, + UnknownSpecialVariable(_) => exit_unknown_special_variable, + UnhandledHIRThrow => exit_unhandled_hir_insn, + UnhandledHIRInvokeBuiltin => exit_unhandled_hir_insn, + UnhandledHIRUnknown(_) => exit_unhandled_hir_insn, + UnhandledYARVInsn(_) => exit_unhandled_yarv_insn, + UnhandledBlockArg => exit_unhandled_block_arg, + FixnumAddOverflow => exit_fixnum_add_overflow, + FixnumSubOverflow => exit_fixnum_sub_overflow, + FixnumMultOverflow => exit_fixnum_mult_overflow, + FixnumLShiftOverflow => exit_fixnum_lshift_overflow, + FixnumModByZero => exit_fixnum_mod_by_zero, + FixnumDivByZero => exit_fixnum_div_by_zero, + BoxFixnumOverflow => exit_box_fixnum_overflow, + GuardType(_) => exit_guard_type_failure, + GuardShape(_) => exit_guard_shape_failure, + ExpandArray => exit_expandarray_failure, + GuardNotFrozen => exit_guard_not_frozen_failure, + GuardNotShared => exit_guard_not_shared_failure, + GuardLess => exit_guard_less_failure, + GuardGreaterEq => exit_guard_greater_eq_failure, + GuardSuperMethodEntry => exit_guard_super_method_entry, + CalleeSideExit => exit_callee_side_exit, + ObjToStringFallback => exit_obj_to_string_fallback, + Interrupt => exit_interrupt, + StackOverflow => exit_stackoverflow, + BlockParamProxyNotIseqOrIfunc => exit_block_param_proxy_not_iseq_or_ifunc, + BlockParamProxyNotNil => exit_block_param_proxy_not_nil, + BlockParamProxyFallbackMiss => exit_block_param_proxy_fallback_miss, + BlockParamProxyProfileNotCovered => exit_block_param_proxy_profile_not_covered, + BlockParamWbRequired => exit_block_param_wb_required, + TooManyKeywordParameters => exit_too_many_keyword_parameters, + SplatKwNotNilOrHash => exit_splatkw_not_nil_or_hash, + SplatKwPolymorphic => exit_splatkw_polymorphic, + SplatKwNotProfiled => exit_splatkw_not_profiled, + DirectiveInduced => exit_directive_induced, + PatchPoint(Invariant::BOPRedefined { .. }) + => exit_patchpoint_bop_redefined, + PatchPoint(Invariant::MethodRedefined { .. }) + => exit_patchpoint_method_redefined, + PatchPoint(Invariant::StableConstantNames { .. }) + => exit_patchpoint_stable_constant_names, + PatchPoint(Invariant::NoTracePoint) + => exit_patchpoint_no_tracepoint, + PatchPoint(Invariant::NoEPEscape(_)) + => exit_patchpoint_no_ep_escape, + PatchPoint(Invariant::SingleRactorMode) + => exit_patchpoint_single_ractor_mode, + PatchPoint(Invariant::NoSingletonClass { .. }) + => exit_patchpoint_no_singleton_class, + PatchPoint(Invariant::RootBoxOnly) + => exit_patchpoint_root_box_only, + SendWhileTracing => exit_send_while_tracing, + NoProfileSend => exit_no_profile_send, + InvokeBlockNotIfunc => exit_invokeblock_not_ifunc, + } +} + +pub fn exit_counter_ptr(reason: crate::hir::SideExitReason) -> *mut u64 { + let counter = side_exit_counter(reason); + counter_ptr(counter) +} + +pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter { + use crate::hir::SendFallbackReason::*; + use crate::stats::Counter::*; + match reason { + SendWithoutBlockPolymorphic => send_fallback_send_without_block_polymorphic, + SendWithoutBlockMegamorphic => send_fallback_send_without_block_megamorphic, + SendWithoutBlockNoProfiles => send_fallback_send_without_block_no_profiles, + SendWithoutBlockCfuncNotVariadic => send_fallback_send_without_block_cfunc_not_variadic, + SendWithoutBlockCfuncArrayVariadic => send_fallback_send_without_block_cfunc_array_variadic, + SendWithoutBlockNotOptimizedMethodType(_) => send_fallback_send_without_block_not_optimized_method_type, + SendWithoutBlockNotOptimizedMethodTypeOptimized(_) + => send_fallback_send_without_block_not_optimized_method_type_optimized, + SendWithoutBlockNotOptimizedNeedPermission + => send_fallback_send_without_block_not_optimized_need_permission, + TooManyArgsForLir => send_fallback_too_many_args_for_lir, + SendWithoutBlockBopRedefined => send_fallback_send_without_block_bop_redefined, + SendWithoutBlockOperandsNotFixnum => send_fallback_send_without_block_operands_not_fixnum, + SendWithoutBlockPolymorphicFallback => send_fallback_send_without_block_polymorphic_fallback, + SendDirectKeywordMismatch => send_fallback_send_without_block_direct_keyword_mismatch, + SendDirectKeywordCountMismatch => send_fallback_send_without_block_direct_keyword_count_mismatch, + SendDirectMissingKeyword => send_fallback_send_without_block_direct_missing_keyword, + SendDirectTooManyKeywords => send_fallback_send_without_block_direct_too_many_keywords, + SendPolymorphic => send_fallback_send_polymorphic, + SendMegamorphic => send_fallback_send_megamorphic, + SendNoProfiles => send_fallback_send_no_profiles, + SendCfuncVariadic => send_fallback_send_cfunc_variadic, + SendCfuncArrayVariadic => send_fallback_send_cfunc_array_variadic, + ComplexArgPass => send_fallback_one_or_more_complex_arg_pass, + UnexpectedKeywordArgs => send_fallback_unexpected_keyword_args, + SingletonClassSeen => send_fallback_singleton_class_seen, + ArgcParamMismatch => send_fallback_argc_param_mismatch, + BmethodNonIseqProc => send_fallback_bmethod_non_iseq_proc, + SendNotOptimizedMethodType(_) => send_fallback_send_not_optimized_method_type, + SendNotOptimizedNeedPermission => send_fallback_send_not_optimized_need_permission, + CCallWithFrameTooManyArgs => send_fallback_ccall_with_frame_too_many_args, + ObjToStringNotString => send_fallback_obj_to_string_not_string, + SuperCallWithBlock => send_fallback_super_call_with_block, + SuperFromBlock => send_fallback_super_from_block, + SuperClassNotFound => send_fallback_super_class_not_found, + SuperComplexArgsPass => send_fallback_super_complex_args_pass, + SuperNoProfiles => send_fallback_super_fallback_no_profile, + SuperNotOptimizedMethodType(_) => send_fallback_super_not_optimized_method_type, + SuperPolymorphic => send_fallback_super_polymorphic, + SuperTargetNotFound => send_fallback_super_target_not_found, + SuperTargetComplexArgsPass => send_fallback_super_target_complex_args_pass, + InvokeBlockNotSpecialized => send_fallback_invokeblock_not_specialized, + SendForwardNotSpecialized => send_fallback_sendforward_not_specialized, + InvokeSuperForwardNotSpecialized => send_fallback_invokesuperforward_not_specialized, + SingleRactorModeRequired => send_fallback_single_ractor_mode_required, + Uncategorized(_) => send_fallback_uncategorized, + } +} + +pub fn send_without_block_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_without_block_def_type_iseq, + Cfunc => unspecialized_send_without_block_def_type_cfunc, + Attrset => unspecialized_send_without_block_def_type_attrset, + Ivar => unspecialized_send_without_block_def_type_ivar, + Bmethod => unspecialized_send_without_block_def_type_bmethod, + Zsuper => unspecialized_send_without_block_def_type_zsuper, + Alias => unspecialized_send_without_block_def_type_alias, + Undefined => unspecialized_send_without_block_def_type_undef, + NotImplemented => unspecialized_send_without_block_def_type_not_implemented, + Optimized => unspecialized_send_without_block_def_type_optimized, + Missing => unspecialized_send_without_block_def_type_missing, + Refined => unspecialized_send_without_block_def_type_refined, + Null => unspecialized_send_without_block_def_type_null, + } +} + +pub fn send_without_block_fallback_counter_for_optimized_method_type(method_type: crate::hir::OptimizedMethodType) -> Counter { + use crate::hir::OptimizedMethodType::*; + use crate::stats::Counter::*; + + match method_type { + Send => unspecialized_send_without_block_def_type_optimized_send, + Call => unspecialized_send_without_block_def_type_optimized_call, + BlockCall => unspecialized_send_without_block_def_type_optimized_block_call, + StructAref => unspecialized_send_without_block_def_type_optimized_struct_aref, + StructAset => unspecialized_send_without_block_def_type_optimized_struct_aset, + } +} + +pub fn send_fallback_counter_for_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_send_def_type_iseq, + Cfunc => unspecialized_send_def_type_cfunc, + Attrset => unspecialized_send_def_type_attrset, + Ivar => unspecialized_send_def_type_ivar, + Bmethod => unspecialized_send_def_type_bmethod, + Zsuper => unspecialized_send_def_type_zsuper, + Alias => unspecialized_send_def_type_alias, + Undefined => unspecialized_send_def_type_undef, + NotImplemented => unspecialized_send_def_type_not_implemented, + Optimized => unspecialized_send_def_type_optimized, + Missing => unspecialized_send_def_type_missing, + Refined => unspecialized_send_def_type_refined, + Null => unspecialized_send_def_type_null, + } +} + +pub fn send_fallback_counter_for_super_method_type(method_type: crate::hir::MethodType) -> Counter { + use crate::hir::MethodType::*; + use crate::stats::Counter::*; + + match method_type { + Iseq => unspecialized_super_def_type_iseq, + Cfunc => unspecialized_super_def_type_cfunc, + Attrset => unspecialized_super_def_type_attrset, + Ivar => unspecialized_super_def_type_ivar, + Bmethod => unspecialized_super_def_type_bmethod, + Zsuper => unspecialized_super_def_type_zsuper, + Alias => unspecialized_super_def_type_alias, + Undefined => unspecialized_super_def_type_undef, + NotImplemented => unspecialized_super_def_type_not_implemented, + Optimized => unspecialized_super_def_type_optimized, + Missing => unspecialized_super_def_type_missing, + Refined => unspecialized_super_def_type_refined, + Null => unspecialized_super_def_type_null, + } +} + +/// Primitive called in zjit.rb. Zero out all the counters. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_reset_stats_bang(_ec: EcPtr, _self: VALUE) -> VALUE { + let counters = ZJITState::get_counters(); + let exit_counters = ZJITState::get_exit_counters(); + + // Reset all counters to zero + *counters = Counters::default(); + + // Reset exit counters for YARV instructions + exit_counters.as_mut_slice().fill(0); + + // Reset send fallback counters + ZJITState::get_send_fallback_counters().as_mut_slice().fill(0); + + // Reset not-inlined counters + ZJITState::get_not_inlined_cfunc_counter_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + // Reset not-annotated counters + ZJITState::get_not_annotated_cfunc_counter_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + // Reset ccall counters + ZJITState::get_ccall_counter_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + // Reset iseq call counters + ZJITState::get_iseq_calls_count_pointers().iter_mut() + .for_each(|b| { **(b.1) = 0; }); + + Qnil +} + /// Return a Hash object that contains ZJIT statistics #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> VALUE { @@ -80,13 +818,24 @@ pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> macro_rules! set_stat { ($hash:ident, $key:expr, $value:expr) => { let key = rust_str_to_sym($key); - // Evaluate $value only when it's needed if key == target_key { - return VALUE::fixnum_from_usize($value as usize); + return $value; } else if $hash != Qnil { #[allow(unused_unsafe)] - unsafe { rb_hash_aset($hash, key, VALUE::fixnum_from_usize($value as usize)); } + unsafe { rb_hash_aset($hash, key, $value); } } + }; + } + + macro_rules! set_stat_usize { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, VALUE::fixnum_from_usize($value as usize)) + } + } + + macro_rules! set_stat_f64 { + ($hash:ident, $key:expr, $value:expr) => { + set_stat!($hash, $key, unsafe { rb_float_new($value) }) } } @@ -95,34 +844,437 @@ pub extern "C" fn rb_zjit_stats(_ec: EcPtr, _self: VALUE, target_key: VALUE) -> } else { Qnil }; - let counters = ZJITState::get_counters(); + // Set default counters for &counter in DEFAULT_COUNTERS { - set_stat!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); } - // Set counters that are enabled when --zjit-stats is enabled - if get_option!(stats) { - set_stat!(hash, "zjit_insns_count", counters.zjit_insns_count); + // Memory usage stats + let code_region_bytes = ZJITState::get_code_block().mapped_region_size(); + set_stat_usize!(hash, "code_region_bytes", code_region_bytes); + set_stat_usize!(hash, "zjit_alloc_bytes", zjit_alloc_bytes()); + set_stat_usize!(hash, "total_mem_bytes", code_region_bytes + zjit_alloc_bytes()); - if unsafe { rb_vm_insns_count } > 0 { - set_stat!(hash, "vm_insns_count", unsafe { rb_vm_insns_count }); - } + // End of default stats. Every counter beyond this is provided only for --zjit-stats. + if !get_option!(stats) { + return hash; + } + + // Set other stats-only counters + for &counter in OTHER_COUNTERS { + set_stat_usize!(hash, &counter.name(), unsafe { *counter_ptr(counter) }); + } + + // Set side-exit counters for each SideExitReason + let mut side_exit_count = 0; + for &counter in EXIT_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + side_exit_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "side_exit_count", side_exit_count); + + // Set side-exit counters for UnhandledYARVInsn + let exit_counters = ZJITState::get_exit_counters(); + for (op_idx, count) in exit_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "unhandled_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Set send fallback counters for each DynamicSendReason + let mut dynamic_send_count = 0; + for &counter in DYNAMIC_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_send_count", dynamic_send_count); + + // Set optimized send counters + let mut optimized_send_count = 0; + for &counter in OPTIMIZED_SEND_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + optimized_send_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "optimized_send_count", optimized_send_count); + set_stat_usize!(hash, "send_count", dynamic_send_count + optimized_send_count); + + // Set send fallback counters for each setivar fallback reason + let mut dynamic_setivar_count = 0; + for &counter in DYNAMIC_SETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_setivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_setivar_count", dynamic_setivar_count); + + // Set send fallback counters for each getivar fallback reason + let mut dynamic_getivar_count = 0; + for &counter in DYNAMIC_GETIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_getivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_getivar_count", dynamic_getivar_count); + + // Set send fallback counters for each definedivar fallback reason + let mut dynamic_definedivar_count = 0; + for &counter in DYNAMIC_DEFINEDIVAR_COUNTERS { + let count = unsafe { *counter_ptr(counter) }; + dynamic_definedivar_count += count; + set_stat_usize!(hash, &counter.name(), count); + } + set_stat_usize!(hash, "dynamic_definedivar_count", dynamic_definedivar_count); + + // Set send fallback counters for Uncategorized + let send_fallback_counters = ZJITState::get_send_fallback_counters(); + for (op_idx, count) in send_fallback_counters.iter().enumerate().take(VM_INSTRUCTION_SIZE as usize) { + let op_name = insn_name(op_idx); + let key_string = "uncategorized_fallback_yarv_insn_".to_owned() + &op_name; + set_stat_usize!(hash, &key_string, *count); + } + + // Only ZJIT_STATS builds support rb_vm_insn_count + if unsafe { rb_vm_insn_count } > 0 { + let vm_insn_count = unsafe { rb_vm_insn_count }; + set_stat_usize!(hash, "vm_insn_count", vm_insn_count); + + let zjit_insn_count = ZJITState::get_counters().zjit_insn_count; + let total_insn_count = vm_insn_count + zjit_insn_count; + set_stat_usize!(hash, "total_insn_count", total_insn_count); + + set_stat_f64!(hash, "ratio_in_zjit", 100.0 * zjit_insn_count as f64 / total_insn_count as f64); + } + + // Set not inlined cfunc counters + let not_inlined_cfuncs = ZJITState::get_not_inlined_cfunc_counter_pointers(); + for (signature, counter) in not_inlined_cfuncs.iter() { + let key_string = format!("not_inlined_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set not annotated cfunc counters + let not_annotated_cfuncs = ZJITState::get_not_annotated_cfunc_counter_pointers(); + for (signature, counter) in not_annotated_cfuncs.iter() { + let key_string = format!("not_annotated_cfuncs_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set ccall counters + let ccall = ZJITState::get_ccall_counter_pointers(); + for (signature, counter) in ccall.iter() { + let key_string = format!("ccall_{signature}"); + set_stat_usize!(hash, &key_string, **counter); + } + + // Set iseq access counters + let iseq_access_counts = ZJITState::get_iseq_calls_count_pointers(); + for (iseq_name, counter) in iseq_access_counts.iter() { + let key_string = format!("iseq_calls_count_{iseq_name}"); + set_stat_usize!(hash, &key_string, **counter); } hash } +pub fn total_exit_count() -> u64 { + EXIT_COUNTERS.iter().fold(0, |sum, counter| sum + unsafe { *counter_ptr(*counter) }) +} + /// Measure the time taken by func() and add that to zjit_compile_time. pub fn with_time_stat<F, R>(counter: Counter, func: F) -> R where F: FnOnce() -> R { let start = Instant::now(); let ret = func(); let nanos = Instant::now().duration_since(start).as_nanos(); - incr_counter(counter, nanos as u64); + incr_counter_by(counter, nanos as u64); ret } /// The number of bytes ZJIT has allocated on the Rust heap. -pub fn zjit_alloc_size() -> usize { - 0 // TODO: report the actual memory usage to support --zjit-mem-size (Shopify/ruby#686) +pub fn zjit_alloc_bytes() -> usize { + jit::GLOBAL_ALLOCATOR.alloc_size.load(Ordering::SeqCst) +} + +/// Record a Perfetto duration event spanning the execution of `func`. +/// Uses Begin/End pairs so nested calls produce properly nested slices. +pub fn trace_compile_phase<F, R>(name: &str, func: F) -> R where F: FnOnce() -> R { + if !get_option!(trace_compiles) { + return func(); + } + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_begin("compile", name, ts, &[]); + } + let result = func(); + if let Some(tracer) = ZJITState::get_tracer() { + let ts = tracer.elapsed_ns(); + tracer.write_duration_end("compile", name, ts); + } + result +} + +/// Fuchsia Trace Format (FXT) binary writer for --zjit-trace-exits. +/// Produces .fxt files that can be opened directly in Perfetto UI. +/// Uses the string table for deduplication of repeated reason/frame strings. +/// See: <https://fuchsia.dev/fuchsia-src/reference/tracing/trace-format> +pub struct PerfettoTracer { + writer: std::io::BufWriter<std::fs::File>, + start_time: std::time::Instant, + event_count: usize, + pub skipped_samples: usize, + /// String table: string content -> interned index (1..32767) + string_table: std::collections::HashMap<String, u16>, + next_string_index: u16, + pid: u32, +} + +impl PerfettoTracer { + /// Write a single 64-bit little-endian word. + fn write_word(&mut self, val: u64) { + use std::io::Write; + let _ = self.writer.write_all(&val.to_le_bytes()); + } + + /// Write bytes padded to 8-byte alignment. + fn write_padded_bytes(&mut self, bytes: &[u8]) { + use std::io::Write; + let _ = self.writer.write_all(bytes); + let remainder = bytes.len() % 8; + if remainder != 0 { + let _ = self.writer.write_all(&[0u8; 7][..8 - remainder]); + } + } + + /// Number of 8-byte words needed for `len` bytes (rounded up). + fn word_count(len: usize) -> u64 { + ((len + 7) / 8) as u64 + } + + pub fn new() -> Self { + let pid = std::process::id(); + let path = format!("/tmp/perfetto-{pid}.fxt"); + let tracer = Self::create(&path, pid); + eprintln!("ZJIT: writing trace exits to {path}"); + tracer + } + + fn create(path: &str, pid: u32) -> Self { + let file = std::fs::File::create(path) + .unwrap_or_else(|e| panic!("ZJIT: failed to create {path}: {e}")); + let mut tracer = PerfettoTracer { + writer: std::io::BufWriter::new(file), + start_time: std::time::Instant::now(), + event_count: 0, + skipped_samples: 0, + string_table: std::collections::HashMap::new(), + next_string_index: 1, // index 0 = empty string + pid, + }; + + // Magic number record: metadata type=4 (trace info), trace info type=0, + // magic=0x16547846 at bits [24..55] + tracer.write_word((1u64 << 4) | (4u64 << 16) | (0x16547846u64 << 24)); + + // Initialization record: 1 tick = 1 nanosecond + tracer.write_word(1u64 | (2u64 << 4)); + tracer.write_word(1_000_000_000u64); + + // Register thread at index 1: (process_koid=pid, thread_koid=1) + tracer.write_word(3u64 | (3u64 << 4) | (1u64 << 16)); + tracer.write_word(pid as u64); + tracer.write_word(1u64); + + // Kernel object record for process: type=7, obj_type=1 (ZX_OBJ_TYPE_PROCESS), no args + let process_name_ref = tracer.intern_string("ruby"); + let ko_process_header: u64 = 7u64 + | (2u64 << 4) // size = 2 words + | (1u64 << 16) // obj_type = ZX_OBJ_TYPE_PROCESS + | ((process_name_ref as u64) << 24); // name + tracer.write_word(ko_process_header); + tracer.write_word(pid as u64); // koid = process id + + // Kernel object record for thread: type=7, obj_type=2 (ZX_OBJ_TYPE_THREAD), 1 arg + let thread_name_ref = tracer.intern_string("main"); + let process_arg_name_ref = tracer.intern_string("process"); + let ko_thread_header: u64 = 7u64 + | (4u64 << 4) // size = 4 words (header + koid + 2-word arg) + | (2u64 << 16) // obj_type = ZX_OBJ_TYPE_THREAD + | ((thread_name_ref as u64) << 24) // name + | (1u64 << 40); // n_args = 1 + tracer.write_word(ko_thread_header); + tracer.write_word(1u64); // koid = thread id (matches thread record) + // Koid argument: type=8, size=2, name="process", value=pid + let arg_header: u64 = 8u64 | (2u64 << 4) | ((process_arg_name_ref as u64) << 16); + tracer.write_word(arg_header); + tracer.write_word(pid as u64); + + // Pre-intern common strings + tracer.intern_string("side_exit"); + tracer.intern_string("compile"); + tracer.intern_string("invalidation"); + // Pre-intern argument names "0".."14" for per-frame arguments + for i in 0..15u32 { + tracer.intern_string(&i.to_string()); + } + + // Flush header immediately so something is written even if process exits abruptly + { + use std::io::Write; + let _ = tracer.writer.flush(); + } + + tracer + } + + /// Intern a string into the string table, writing a string record if new. + /// Returns the string table index (1..32767). Returns 0 for empty strings + /// or if the table is full. + fn intern_string(&mut self, s: &str) -> u16 { + if s.is_empty() { + return 0; + } + if let Some(&idx) = self.string_table.get(s) { + return idx; + } + if self.next_string_index >= 0x8000 { + return 0; // table full + } + + let idx = self.next_string_index; + let bytes = s.as_bytes(); + let len = bytes.len().min(0x7FFF); // 15-bit max length + let record_words = 1 + Self::word_count(len); + + // String record: type=2, index in [16..30], length in [32..46] + let header: u64 = 2u64 + | (record_words << 4) + | ((idx as u64) << 16) + | ((len as u64) << 32); + self.write_word(header); + self.write_padded_bytes(&bytes[..len]); + + self.string_table.insert(s.to_string(), idx); + self.next_string_index += 1; + idx + } + + /// Return nanoseconds elapsed since tracer creation. + pub fn elapsed_ns(&self) -> u64 { + self.start_time.elapsed().as_nanos() as u64 + } + + /// Write a Duration Begin event (FXT event type 2) with optional frame arguments. + pub fn write_duration_begin(&mut self, category: &str, name: &str, ts_ns: u64, frames: &[String]) { + self.write_duration_event(2, category, name, ts_ns, frames); + } + + /// Write a Duration End event (FXT event type 3). + pub fn write_duration_end(&mut self, category: &str, name: &str, ts_ns: u64) { + self.write_duration_event(3, category, name, ts_ns, &[]); + } + + /// Write a Duration Begin or End event with optional frame arguments. + fn write_duration_event(&mut self, event_type: u64, category: &str, name: &str, ts_ns: u64, frames: &[String]) { + let category_ref = self.intern_string(category); + let name_ref = self.intern_string(name); + + let n_args = frames.len().min(15) as u64; + let mut frame_refs: Vec<(u16, u16)> = Vec::with_capacity(n_args as usize); + for (i, frame) in frames.iter().take(15).enumerate() { + let fname_ref = self.intern_string(&i.to_string()); + let value_ref = self.intern_string(frame); + frame_refs.push((fname_ref, value_ref)); + } + + let event_words: u64 = 2 + n_args; + let header: u64 = 4u64 // record type = event + | (event_words << 4) // record size + | (event_type << 16) // event type = begin or end + | (n_args << 20) // argument count + | (1u64 << 24) // thread_ref = 1 + | ((category_ref as u64) << 32) + | ((name_ref as u64) << 48); + self.write_word(header); + self.write_word(ts_ns); + + for (fname_ref, value_ref) in frame_refs { + let arg_header: u64 = 6u64 + | (1u64 << 4) + | ((fname_ref as u64) << 16) + | ((value_ref as u64) << 32); + self.write_word(arg_header); + } + + self.event_count += 1; + + use std::io::Write; + let _ = self.writer.flush(); + } + + pub fn write_event(&mut self, category: &str, reason: &str, frames: &[String]) { + let ts_nanos = self.start_time.elapsed().as_nanos() as u64; + + // Intern event metadata strings (may emit string records first) + let category_ref = self.intern_string(category); + let name_ref = self.intern_string(reason); + + // Intern each frame label and collect refs (max 15 due to 4-bit n_args) + let n_args = frames.len().min(15) as u64; + let mut frame_refs: Vec<(u16, u16)> = Vec::with_capacity(n_args as usize); + for (i, frame) in frames.iter().take(15).enumerate() { + let name_ref = self.intern_string(&i.to_string()); + let value_ref = self.intern_string(frame); + frame_refs.push((name_ref, value_ref)); + } + + // Each fully-interned string argument is exactly 1 word + let event_words = 2 + n_args; + let header: u64 = 4u64 + | (event_words << 4) + | (n_args << 20) // argument count + | (1u64 << 24) // thread_ref = 1 + | ((category_ref as u64) << 32) + | ((name_ref as u64) << 48); + self.write_word(header); + self.write_word(ts_nanos); + + // One 1-word string argument per frame: type=6, size=1, indexed name, indexed value + for (name_ref, value_ref) in frame_refs { + let arg_header: u64 = 6u64 + | (1u64 << 4) + | ((name_ref as u64) << 16) + | ((value_ref as u64) << 32); + self.write_word(arg_header); + } + + self.event_count += 1; + + // Flush to ensure data reaches disk. Static globals may not be + // dropped on process exit, so we can't rely on Drop for flushing. + use std::io::Write; + let _ = self.writer.flush(); + } +} + +impl Drop for PerfettoTracer { + fn drop(&mut self) { + use std::io::Write; + let _ = self.writer.flush(); + } +} + +/// Primitive called in zjit.rb +/// +/// Check if trace_exits generation is enabled. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_trace_exit_locations_enabled_p(_ec: EcPtr, _ruby_self: VALUE) -> VALUE { + // Builtin zjit.rb calls this even if ZJIT is disabled, so OPTIONS may not be set. + if unsafe { OPTIONS.as_ref() }.is_some_and(|opts| opts.trace_side_exits.is_some()) { + Qtrue + } else { + Qfalse + } } diff --git a/zjit/src/ttycolors.rs b/zjit/src/ttycolors.rs new file mode 100644 index 0000000000..f325772431 --- /dev/null +++ b/zjit/src/ttycolors.rs @@ -0,0 +1,31 @@ +use std::io::IsTerminal; + +pub fn stdout_supports_colors() -> bool { + std::io::stdout().is_terminal() +} + +#[cfg_attr(not(feature = "disasm"), allow(dead_code))] +#[derive(Copy, Clone, Debug)] +pub struct TerminalColor { + pub bold_begin: &'static str, + pub bold_end: &'static str, +} + +pub static TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "\x1b[1m", + bold_end: "\x1b[22m", +}; + +pub static NON_TTY_TERMINAL_COLOR: TerminalColor = TerminalColor { + bold_begin: "", + bold_end: "", +}; + +/// Terminal escape codes for colors, font weight, etc. Only enabled if stdout is a TTY. +pub fn get_colors() -> &'static TerminalColor { + if stdout_supports_colors() { + &TTY_TERMINAL_COLOR + } else { + &NON_TTY_TERMINAL_COLOR + } +} diff --git a/zjit/src/virtualmem.rs b/zjit/src/virtualmem.rs index 42ce525fde..0088ef1a66 100644 --- a/zjit/src/virtualmem.rs +++ b/zjit/src/virtualmem.rs @@ -4,15 +4,11 @@ // benefit. use std::ptr::NonNull; +use crate::cruby::*; +use crate::stats::zjit_alloc_bytes; -use crate::stats::zjit_alloc_size; - -#[cfg(not(test))] pub type VirtualMem = VirtualMemory<sys::SystemAllocator>; -#[cfg(test)] -pub type VirtualMem = VirtualMemory<tests::TestingAllocator>; - /// Memory for generated executable machine code. When not testing, we reserve address space for /// the entire region upfront and map physical memory into the reserved address space as needed. On /// Linux, this is basically done using an `mmap` with `PROT_NONE` upfront and gradually using @@ -30,7 +26,7 @@ pub struct VirtualMemory<A: Allocator> { region_size_bytes: usize, /// mapped_region_bytes + zjit_alloc_size may not increase beyond this limit. - memory_limit_bytes: usize, + memory_limit_bytes: Option<usize>, /// Number of bytes per "page", memory protection permission can only be controlled at this /// granularity. @@ -90,7 +86,7 @@ impl CodePtr { /// Get the address of the code pointer. pub fn raw_addr(self, base: &impl CodePtrBase) -> usize { - self.raw_ptr(base) as usize + self.raw_ptr(base).addr() } /// Get the offset component for the code pointer. Useful finding the distance between two @@ -110,6 +106,28 @@ pub enum WriteError { use WriteError::*; +impl VirtualMem { + /// Allocate a VirtualMem insntace with a requested size + pub fn alloc(exec_mem_bytes: usize, mem_bytes: Option<usize>) -> Self { + let virt_block: *mut u8 = unsafe { rb_jit_reserve_addr_space(exec_mem_bytes as u32) }; + + // Memory protection syscalls need page-aligned addresses, so check it here. Assuming + // `virt_block` is page-aligned, `second_half` should be page-aligned as long as the + // page size in bytes is a power of two 2^19 or smaller. This is because the user + // requested size is half of mem_option * 2^20 as it's in MiB. + // + // Basically, we don't support x86-64 2MiB and 1GiB pages. ARMv8 can do up to 64KiB + // (2^16 bytes) pages, which should be fine. 4KiB pages seem to be the most popular though. + let page_size = unsafe { rb_jit_get_page_size() }; + assert_eq!( + virt_block as usize % page_size as usize, 0, + "Start of virtual address block should be page-aligned", + ); + + Self::new(sys::SystemAllocator {}, page_size, NonNull::new(virt_block).unwrap(), exec_mem_bytes, mem_bytes) + } +} + impl<A: Allocator> VirtualMemory<A> { /// Bring a part of the address space under management. pub fn new( @@ -117,7 +135,7 @@ impl<A: Allocator> VirtualMemory<A> { page_size: u32, virt_region_start: NonNull<u8>, region_size_bytes: usize, - memory_limit_bytes: usize, + memory_limit_bytes: Option<usize>, ) -> Self { assert_ne!(0, page_size); let page_size_bytes = page_size as usize; @@ -178,6 +196,12 @@ impl<A: Allocator> VirtualMemory<A> { let whole_region_end = start.wrapping_add(self.region_size_bytes); let alloc = &mut self.allocator; + // Ignore zjit_alloc_size() if self.memory_limit_bytes is None for testing + let mut required_region_bytes = page_addr + page_size - start as usize; + if self.memory_limit_bytes.is_some() { + required_region_bytes += zjit_alloc_bytes(); + } + assert!((start..=whole_region_end).contains(&mapped_region_end)); if (start..mapped_region_end).contains(&raw) { @@ -190,7 +214,7 @@ impl<A: Allocator> VirtualMemory<A> { self.current_write_page = Some(page_addr); } else if (start..whole_region_end).contains(&raw) && - (page_addr + page_size - start as usize) + zjit_alloc_size() < self.memory_limit_bytes { + required_region_bytes < self.memory_limit_bytes.unwrap_or(self.region_size_bytes) { // Writing to a brand new page let mapped_region_end_addr = mapped_region_end as usize; let alloc_size = page_addr - mapped_region_end_addr + page_size; @@ -234,6 +258,29 @@ impl<A: Allocator> VirtualMemory<A> { Ok(()) } + /// Return true if write_byte() can allocate a new page + pub fn can_allocate(&self) -> bool { + let memory_usage_bytes = self.mapped_region_bytes + zjit_alloc_bytes(); + let memory_limit_bytes = self.memory_limit_bytes.unwrap_or(self.region_size_bytes); + memory_usage_bytes + self.page_size_bytes < memory_limit_bytes + } + + /// Make all the code in the region writable. Call this before bulk writes (e.g. GC + /// reference updates). See [Self] for usual usage flow. + pub fn mark_all_writable(&mut self) { + self.current_write_page = None; + + let region_start = self.region_start; + let mapped_region_bytes: u32 = self.mapped_region_bytes.try_into().unwrap(); + + // Make mapped region writable + if mapped_region_bytes > 0 { + if !self.allocator.mark_writable(region_start.as_ptr(), mapped_region_bytes) { + panic!("Cannot make JIT memory region writable"); + } + } + } + /// Make all the code in the region executable. Call this at the end of a write session. /// See [Self] for usual usage flow. pub fn mark_all_executable(&mut self) { @@ -276,7 +323,6 @@ impl<A: Allocator> CodePtrBase for VirtualMemory<A> { } /// Requires linking with CRuby to work -#[cfg(not(test))] pub mod sys { use crate::cruby::*; @@ -287,15 +333,15 @@ pub mod sys { impl super::Allocator for SystemAllocator { fn mark_writable(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_writable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_writable(ptr as VoidPtr, size) } } fn mark_executable(&mut self, ptr: *const u8, size: u32) { - unsafe { rb_zjit_mark_executable(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_executable(ptr as VoidPtr, size) } } fn mark_unused(&mut self, ptr: *const u8, size: u32) -> bool { - unsafe { rb_zjit_mark_unused(ptr as VoidPtr, size) } + unsafe { rb_jit_mark_unused(ptr as VoidPtr, size) } } } } @@ -369,6 +415,12 @@ pub mod tests { // Fictional architecture where each page is 4 bytes long const PAGE_SIZE: usize = 4; fn new_dummy_virt_mem() -> VirtualMemory<TestingAllocator> { + unsafe { + if crate::options::OPTIONS.is_none() { + crate::options::OPTIONS = Some(crate::options::Options::default()); + } + } + let mem_size = PAGE_SIZE * 10; let alloc = TestingAllocator::new(mem_size); let mem_start: *const u8 = alloc.mem_start(); @@ -378,7 +430,7 @@ pub mod tests { PAGE_SIZE.try_into().unwrap(), NonNull::new(mem_start as *mut u8).unwrap(), mem_size, - 128 * 1024 * 1024, + None, ) } |